Enforce only major version in JSON model schema

* Remove f-string, since it's not supported by Python 3.5

* Add Python 3.5 to CI, to ensure compatibility

* Remove duplicated matplotlib

* Show deprecation notice for Python 3.5

* Fix lint

* Fix lint

.clang-tidy

@@ -1,4 +1,4 @@
-Checks: 'modernize-*,-modernize-make-*,-modernize-use-auto,-modernize-raw-string-literal,google-*,-google-default-arguments,-clang-diagnostic-#pragma-messages,readability-identifier-naming'
+Checks: 'modernize-*,-modernize-make-*,-modernize-use-auto,-modernize-raw-string-literal,-modernize-avoid-c-arrays,-modernize-use-trailing-return-type,google-*,-google-default-arguments,-clang-diagnostic-#pragma-messages,readability-identifier-naming'
 CheckOptions:
   - { key: readability-identifier-naming.ClassCase,           value: CamelCase  }
   - { key: readability-identifier-naming.StructCase,          value: CamelCase  }

.gitignore

@@ -17,7 +17,7 @@
 *.tar.gz
 *conf
 *buffer
-*model
+*.model
 *pyc
 *.train
 *.test
@@ -69,10 +69,8 @@ config.mk
 /xgboost
 *.data
 build_plugin
-.idea
 recommonmark/
 tags
-*.iml
 *.class
 target
 *.swp
@@ -90,9 +88,16 @@ lib/
 # spark
 metastore_db
 
-plugin/updater_gpu/test/cpp/data
 /include/xgboost/build_config.h
 
 # files from R-package source install
 **/config.status
 R-package/src/Makevars
+
+# Visual Studio Code
+/.vscode/
+
+# IntelliJ/CLion
+.idea
+*.iml
+/cmake-build-debug/

.travis.yml

@@ -1,36 +1,51 @@
 # disable sudo for container build.
 sudo: required
 
-# Enabling test on Linux and OS X
+# Enabling test OS X
 os:
+  - linux
   - osx
 
-osx_image: xcode9.3
+osx_image: xcode10.3
+dist: bionic
 
 # Use Build Matrix to do lint and build seperately
 env:
   matrix:
     # python package test
     - TASK=python_test
+    # test installation of Python source distribution
+    - TASK=python_sdist_test
     # java package test
     - TASK=java_test
     # cmake test
-    # - TASK=cmake_test
+    - TASK=cmake_test
 
-# dependent apt packages
+matrix:
+  exclude:
+    - os: linux
+      env: TASK=python_test
+    - os: linux
+      env: TASK=java_test
+    - os: linux
+      env: TASK=cmake_test
+
+# dependent brew packages
 addons:
   homebrew:
     packages:
-      - gcc@7
+      - cmake
+      - libomp
       - graphviz
       - openssl
       - libgit2
+      - wget
       - r
     update: true
 
 before_install:
   - source dmlc-core/scripts/travis/travis_setup_env.sh
-  - export PYTHONPATH=${PYTHONPATH}:${PWD}/python-package
+  - if [ "${TASK}" != "python_sdist_test" ]; then export PYTHONPATH=${PYTHONPATH}:${PWD}/python-package; fi
   - echo "MAVEN_OPTS='-Xmx2g -XX:MaxPermSize=1024m -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=error'" > ~/.mavenrc
 
 install:

CMakeLists.txt

@@ -1,14 +1,23 @@
-cmake_minimum_required(VERSION 3.3)
-project(xgboost LANGUAGES CXX C VERSION 0.90)
+cmake_minimum_required(VERSION 3.12)
+project(xgboost LANGUAGES CXX C VERSION 1.0.1)
 include(cmake/Utils.cmake)
-list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
+list(APPEND CMAKE_MODULE_PATH "${xgboost_SOURCE_DIR}/cmake/modules")
 cmake_policy(SET CMP0022 NEW)
 
-message(STATUS "CMake version ${CMAKE_VERSION}")
-if (MSVC)
-  cmake_minimum_required(VERSION 3.11)
-endif (MSVC)
+if ((${CMAKE_VERSION} VERSION_GREATER 3.13) OR (${CMAKE_VERSION} VERSION_EQUAL 3.13))
+  cmake_policy(SET CMP0077 NEW)
+endif ((${CMAKE_VERSION} VERSION_GREATER 3.13) OR (${CMAKE_VERSION} VERSION_EQUAL 3.13))
 
+message(STATUS "CMake version ${CMAKE_VERSION}")
+
+if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
+  message(FATAL_ERROR "GCC version must be at least 5.0!")
+endif()
+
+include(${xgboost_SOURCE_DIR}/cmake/FindPrefetchIntrinsics.cmake)
+find_prefetch_intrinsics()
+include(${xgboost_SOURCE_DIR}/cmake/Version.cmake)
+write_version()
 set_default_configuration_release()
 
 #-- Options
@@ -18,44 +27,61 @@ option(USE_OPENMP "Build with OpenMP support." ON)
 option(JVM_BINDINGS "Build JVM bindings" OFF)
 option(R_LIB "Build shared library for R package" OFF)
 ## Dev
+option(USE_DEBUG_OUTPUT "Dump internal training results like gradients and predictions to stdout.
+Should only be used for debugging." OFF)
 option(GOOGLE_TEST "Build google tests" OFF)
-option(USE_DMLC_GTEST "Use google tests bundled with dmlc-core submodule (EXPERIMENTAL)" OFF)
+option(USE_DMLC_GTEST "Use google tests bundled with dmlc-core submodule" OFF)
 option(USE_NVTX "Build with cuda profiling annotations. Developers only." OFF)
 set(NVTX_HEADER_DIR "" CACHE PATH "Path to the stand-alone nvtx header")
+option(RABIT_MOCK "Build rabit with mock" OFF)
 ## CUDA
 option(USE_CUDA  "Build with GPU acceleration" OFF)
-option(USE_NCCL  "Build with NCCL to enable multi-GPU support." OFF)
+option(USE_NCCL  "Build with NCCL to enable distributed GPU support." OFF)
 option(BUILD_WITH_SHARED_NCCL "Build with shared NCCL library." OFF)
 set(GPU_COMPUTE_VER "" CACHE STRING
   "Semicolon separated list of compute versions to be built against, e.g. '35;61'")
-if (BUILD_WITH_SHARED_NCCL AND (NOT USE_NCCL))
-  message(SEND_ERROR "Build XGBoost with -DUSE_NCCL=ON to enable BUILD_WITH_SHARED_NCCL.")
-endif (BUILD_WITH_SHARED_NCCL AND (NOT USE_NCCL))
+## Copied From dmlc
+option(USE_HDFS "Build with HDFS support" OFF)
+option(USE_AZURE "Build with AZURE support" OFF)
+option(USE_S3 "Build with S3 support" OFF)
 ## Sanitizers
 option(USE_SANITIZER "Use santizer flags" OFF)
 option(SANITIZER_PATH "Path to sanitizes.")
 set(ENABLED_SANITIZERS "address" "leak" CACHE STRING
   "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
-address, leak and thread.")
+address, leak, undefined and thread.")
 ## Plugins
 option(PLUGIN_LZ4 "Build lz4 plugin" OFF)
 option(PLUGIN_DENSE_PARSER "Build dense parser plugin" OFF)
 
-## Deprecation warning
+#-- Checks for building XGBoost
+if (USE_DEBUG_OUTPUT AND (NOT (CMAKE_BUILD_TYPE MATCHES Debug)))
+  message(SEND_ERROR "Do not enable `USE_DEBUG_OUTPUT' with release build.")
+endif (USE_DEBUG_OUTPUT AND (NOT (CMAKE_BUILD_TYPE MATCHES Debug)))
+if (USE_NCCL AND NOT (USE_CUDA))
+  message(SEND_ERROR "`USE_NCCL` must be enabled with `USE_CUDA` flag.")
+endif (USE_NCCL AND NOT (USE_CUDA))
+if (BUILD_WITH_SHARED_NCCL AND (NOT USE_NCCL))
+  message(SEND_ERROR "Build XGBoost with -DUSE_NCCL=ON to enable BUILD_WITH_SHARED_NCCL.")
+endif (BUILD_WITH_SHARED_NCCL AND (NOT USE_NCCL))
+if (JVM_BINDINGS AND R_LIB)
+  message(SEND_ERROR "`R_LIB' is not compatible with `JVM_BINDINGS' as they both have customized configurations.")
+endif (JVM_BINDINGS AND R_LIB)
+if (R_LIB AND GOOGLE_TEST)
+  message(WARNING "Some C++ unittests will fail with `R_LIB` enabled,
+ as R package redirects some functions to R runtime implementation.")
+endif (R_LIB AND GOOGLE_TEST)
 if (USE_AVX)
-  message(WARNING "The option 'USE_AVX' is deprecated as experimental AVX features have been removed from xgboost.")
+  message(SEND_ERROR  "The option 'USE_AVX' is deprecated as experimental AVX features have been removed from XGBoost.")
 endif (USE_AVX)
 
-# Sanitizer
+#-- Sanitizer
 if (USE_SANITIZER)
-  # Older CMake versions have had troubles with Sanitizer
-  cmake_minimum_required(VERSION 3.12)
   include(cmake/Sanitizer.cmake)
   enable_sanitizers("${ENABLED_SANITIZERS}")
 endif (USE_SANITIZER)
 
 if (USE_CUDA)
-  cmake_minimum_required(VERSION 3.12)
   SET(USE_OPENMP ON CACHE BOOL "CUDA requires OpenMP" FORCE)
   # `export CXX=' is ignored by CMake CUDA.
   set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
@@ -67,9 +93,18 @@ if (USE_CUDA)
   message(STATUS "CUDA GEN_CODE: ${GEN_CODE}")
 endif (USE_CUDA)
 
+if (USE_OPENMP)
+  if (APPLE)
+    # Require CMake 3.16+ on Mac OSX, as previous versions of CMake had trouble locating
+    # OpenMP on Mac. See https://github.com/dmlc/xgboost/pull/5146#issuecomment-568312706
+    cmake_minimum_required(VERSION 3.16)
+  endif (APPLE)
+  find_package(OpenMP REQUIRED)
+endif (USE_OPENMP)
+
 # dmlc-core
 msvc_use_static_runtime()
-add_subdirectory(${PROJECT_SOURCE_DIR}/dmlc-core)
+add_subdirectory(${xgboost_SOURCE_DIR}/dmlc-core)
 set_target_properties(dmlc PROPERTIES
   CXX_STANDARD 11
   CXX_STANDARD_REQUIRED ON
@@ -77,40 +112,29 @@ set_target_properties(dmlc PROPERTIES
 list(APPEND LINKED_LIBRARIES_PRIVATE dmlc)
 
 # rabit
-# full rabit doesn't build on windows, so we can't import it as subdirectory
-if(MINGW OR R_LIB)
-  set(RABIT_SOURCES
-    rabit/src/engine_empty.cc
-    rabit/src/c_api.cc)
-else ()
-  set(RABIT_SOURCES
-    rabit/src/allreduce_base.cc
-    rabit/src/allreduce_robust.cc
-    rabit/src/engine.cc
-    rabit/src/c_api.cc)
-endif (MINGW OR R_LIB)
-add_library(rabit STATIC ${RABIT_SOURCES})
-target_include_directories(rabit PRIVATE
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/dmlc-core/include>
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/rabit/include/rabit>)
-set_target_properties(rabit
-  PROPERTIES
-  CXX_STANDARD 11
-  CXX_STANDARD_REQUIRED ON
-  POSITION_INDEPENDENT_CODE ON)
-list(APPEND LINKED_LIBRARIES_PRIVATE rabit)
+set(RABIT_BUILD_DMLC OFF)
+set(DMLC_ROOT ${xgboost_SOURCE_DIR}/dmlc-core)
+set(RABIT_WITH_R_LIB ${R_LIB})
+add_subdirectory(rabit)
+
+if (RABIT_MOCK)
+  list(APPEND LINKED_LIBRARIES_PRIVATE rabit_mock_static)
+else()
+  list(APPEND LINKED_LIBRARIES_PRIVATE rabit)
+endif(RABIT_MOCK)
 
 # Exports some R specific definitions and objects
 if (R_LIB)
-  add_subdirectory(${PROJECT_SOURCE_DIR}/R-package)
+  add_subdirectory(${xgboost_SOURCE_DIR}/R-package)
 endif (R_LIB)
 
 # core xgboost
-add_subdirectory(${PROJECT_SOURCE_DIR}/src)
+add_subdirectory(${xgboost_SOURCE_DIR}/plugin)
+add_subdirectory(${xgboost_SOURCE_DIR}/src)
 set(XGBOOST_OBJ_SOURCES "${XGBOOST_OBJ_SOURCES};$<TARGET_OBJECTS:objxgboost>")
 
 #-- Shared library
-add_library(xgboost SHARED ${XGBOOST_OBJ_SOURCES} ${PLUGINS_SOURCES})
+add_library(xgboost SHARED ${XGBOOST_OBJ_SOURCES})
 target_include_directories(xgboost
   INTERFACE
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>
@@ -119,22 +143,18 @@ target_link_libraries(xgboost PRIVATE ${LINKED_LIBRARIES_PRIVATE})
 
 # This creates its own shared library `xgboost4j'.
 if (JVM_BINDINGS)
-  add_subdirectory(${PROJECT_SOURCE_DIR}/jvm-packages)
+  add_subdirectory(${xgboost_SOURCE_DIR}/jvm-packages)
 endif (JVM_BINDINGS)
 #-- End shared library
 
 #-- CLI for xgboost
-add_executable(runxgboost ${PROJECT_SOURCE_DIR}/src/cli_main.cc ${XGBOOST_OBJ_SOURCES})
-# For cli_main.cc only
-if (USE_OPENMP)
-  find_package(OpenMP REQUIRED)
-  target_compile_options(runxgboost PRIVATE ${OpenMP_CXX_FLAGS})
-endif (USE_OPENMP)
+add_executable(runxgboost ${xgboost_SOURCE_DIR}/src/cli_main.cc ${XGBOOST_OBJ_SOURCES})
+
 target_include_directories(runxgboost
   PRIVATE
-  ${PROJECT_SOURCE_DIR}/include
-  ${PROJECT_SOURCE_DIR}/dmlc-core/include
-  ${PROJECT_SOURCE_DIR}/rabit/include)
+  ${xgboost_SOURCE_DIR}/include
+  ${xgboost_SOURCE_DIR}/dmlc-core/include
+  ${xgboost_SOURCE_DIR}/rabit/include)
 target_link_libraries(runxgboost PRIVATE ${LINKED_LIBRARIES_PRIVATE})
 set_target_properties(
   runxgboost PROPERTIES
@@ -143,8 +163,8 @@ set_target_properties(
   CXX_STANDARD_REQUIRED ON)
 #-- End CLI for xgboost
 
-set_output_directory(runxgboost ${PROJECT_SOURCE_DIR})
-set_output_directory(xgboost ${PROJECT_SOURCE_DIR}/lib)
+set_output_directory(runxgboost ${xgboost_SOURCE_DIR})
+set_output_directory(xgboost ${xgboost_SOURCE_DIR}/lib)
 # Ensure these two targets do not build simultaneously, as they produce outputs with conflicting names
 add_dependencies(xgboost runxgboost)
 
@@ -167,11 +187,9 @@ if (BUILD_C_DOC)
 endif (BUILD_C_DOC)
 
 include(GNUInstallDirs)
-# Exposing only C APIs.
-install(FILES
-  "${PROJECT_SOURCE_DIR}/include/xgboost/c_api.h"
-  DESTINATION
-  include/xgboost/)
+# Install all headers.  Please note that currently the C++ headers does not form an "API".
+install(DIRECTORY ${xgboost_SOURCE_DIR}/include/xgboost
+  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 install(TARGETS xgboost runxgboost
   EXPORT XGBoostTargets
@@ -203,21 +221,21 @@ install(
 if (GOOGLE_TEST)
   enable_testing()
   # Unittests.
-  add_subdirectory(${PROJECT_SOURCE_DIR}/tests/cpp)
+  add_subdirectory(${xgboost_SOURCE_DIR}/tests/cpp)
   add_test(
     NAME TestXGBoostLib
     COMMAND testxgboost
-    WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+    WORKING_DIRECTORY ${xgboost_BINARY_DIR})
 
   # CLI tests
   configure_file(
-    ${PROJECT_SOURCE_DIR}/tests/cli/machine.conf.in
-    ${PROJECT_BINARY_DIR}/tests/cli/machine.conf
+    ${xgboost_SOURCE_DIR}/tests/cli/machine.conf.in
+    ${xgboost_BINARY_DIR}/tests/cli/machine.conf
     @ONLY)
   add_test(
     NAME TestXGBoostCLI
-    COMMAND runxgboost ${PROJECT_BINARY_DIR}/tests/cli/machine.conf
-    WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+    COMMAND runxgboost ${xgboost_BINARY_DIR}/tests/cli/machine.conf
+    WORKING_DIRECTORY ${xgboost_BINARY_DIR})
   set_tests_properties(TestXGBoostCLI
     PROPERTIES
     PASS_REGULAR_EXPRESSION ".*test-rmse:0.087.*")

CONTRIBUTORS.md

@@ -2,34 +2,42 @@ Contributors of DMLC/XGBoost
 ============================
 XGBoost has been developed and used by a group of active community. Everyone is more than welcomed to is a great way to make the project better and more accessible to more users.
 
-Committers
+Project Management Committee(PMC) 
 ----------
-Committers are people who have made substantial contribution to the project and granted write access to the project.
+The Project Management Committee(PMC) consists group of active committers that moderate the discussion, manage the project release, and proposes new committer/PMC members. 
+
 * [Tianqi Chen](https://github.com/tqchen), University of Washington
   - Tianqi is a Ph.D. student working on large-scale machine learning. He is the creator of the project.
-* [Tong He](https://github.com/hetong007), Amazon AI
-  - Tong is an applied scientist in Amazon AI. He is the maintainer of XGBoost R package.
-* [Vadim Khotilovich](https://github.com/khotilov)
-  - Vadim contributes many improvements in R and core packages.
-* [Bing Xu](https://github.com/antinucleon)
-  - Bing is the original creator of XGBoost Python package and currently the maintainer of [XGBoost.jl](https://github.com/antinucleon/XGBoost.jl).
 * [Michael Benesty](https://github.com/pommedeterresautee)
   - Michael is a lawyer and data scientist in France. He is the creator of XGBoost interactive analysis module in R.
 * [Yuan Tang](https://github.com/terrytangyuan), Ant Financial
   - Yuan is a software engineer in Ant Financial. He contributed mostly in R and Python packages.
 * [Nan Zhu](https://github.com/CodingCat), Uber
   - Nan is a software engineer in Uber. He contributed mostly in JVM packages.
-* [Sergei Lebedev](https://github.com/superbobry), Criteo
-  - Sergei is a software engineer in Criteo. He contributed mostly in JVM packages.
-* [Hongliang Liu](https://github.com/phunterlau)
-* [Scott Lundberg](http://scottlundberg.com/), University of Washington
-  - Scott is a Ph.D. student at University of Washington. He is the creator of SHAP, a unified approach to explain the output of machine learning models such as decision tree ensembles. He also helps maintain the XGBoost Julia package.
+* [Jiaming Yuan](https://github.com/trivialfis)
+  - Jiaming contributed to the GPU algorithms. He has also introduced new abstractions to improve the quality of the C++ codebase.
+* [Hyunsu Cho](http://hyunsu-cho.io/), Amazon AI
+  - Hyunsu is an applied scientist in Amazon AI. He is the maintainer of the XGBoost Python package. He also manages the Jenkins continuous integration system (https://xgboost-ci.net/). He is the initial author of the CPU 'hist' updater.   
 * [Rory Mitchell](https://github.com/RAMitchell), University of Waikato
   - Rory is a Ph.D. student at University of Waikato. He is the original creator of the GPU training algorithms. He improved the CMake build system and continuous integration. 
-* [Hyunsu Cho](http://hyunsu-cho.io/), Amazon AI
-  - Hyunsu is an applied scientist in Amazon AI. He is the maintainer of the XGBoost Python package. He also manages the Jenkins continuous integration system (https://xgboost-ci.net/). He is the initial author of the CPU 'hist' updater. 
-* [Jiaming](https://github.com/trivialfis)
-  - Jiaming contributed to the GPU algorithms. He has also introduced new abstractions to improve the quality of the C++ codebase.
+* [Hongliang Liu](https://github.com/phunterlau)
+
+
+Committers
+----------
+Committers are people who have made substantial contribution to the project and granted write access to the project.
+
+* [Tong He](https://github.com/hetong007), Amazon AI
+  - Tong is an applied scientist in Amazon AI. He is the maintainer of XGBoost R package.
+* [Vadim Khotilovich](https://github.com/khotilov)
+  - Vadim contributes many improvements in R and core packages.
+* [Bing Xu](https://github.com/antinucleon)
+  - Bing is the original creator of XGBoost Python package and currently the maintainer of [XGBoost.jl](https://github.com/antinucleon/XGBoost.jl).
+* [Sergei Lebedev](https://github.com/superbobry), Criteo
+  - Sergei is a software engineer in Criteo. He contributed mostly in JVM packages.
+* [Scott Lundberg](http://scottlundberg.com/), University of Washington
+  - Scott is a Ph.D. student at University of Washington. He is the creator of SHAP, a unified approach to explain the output of machine learning models such as decision tree ensembles. He also helps maintain the XGBoost Julia package.
+
 
 Become a Committer
 ------------------
@@ -89,3 +97,8 @@ List of Contributors
 * [Sam Wilkinson](https://samwilkinson.io)
 * [Matthew Jones](https://github.com/mt-jones)
 * [Jiaxiang Li](https://github.com/JiaxiangBU)
+* [Bryan Woods](https://github.com/bryan-woods)
+  - Bryan added support for cross-validation for the ranking objective
+* [Haoda Fu](https://github.com/fuhaoda)
+* [Evan Kepner](https://github.com/EvanKepner)
+  - Evan Kepner added support for os.PathLike file paths in Python

Jenkinsfile

@@ -6,19 +6,25 @@
 // Command to run command inside a docker container
 dockerRun = 'tests/ci_build/ci_build.sh'
 
+import groovy.transform.Field
+
+@Field
+def commit_id   // necessary to pass a variable from one stage to another
+
 pipeline {
   // Each stage specify its own agent
   agent none
 
   environment {
-    DOCKER_CACHE_REPO = '492475357299.dkr.ecr.us-west-2.amazonaws.com'
+    DOCKER_CACHE_ECR_ID = '492475357299'
+    DOCKER_CACHE_ECR_REGION = 'us-west-2'
   }
 
   // Setup common job properties
   options {
     ansiColor('xterm')
     timestamps()
-    timeout(time: 120, unit: 'MINUTES')
+    timeout(time: 240, unit: 'MINUTES')
     buildDiscarder(logRotator(numToKeepStr: '10'))
     preserveStashes()
   }
@@ -30,6 +36,7 @@ pipeline {
       steps {
         script {
           checkoutSrcs()
+          commit_id = "${GIT_COMMIT}"
         }
         stash name: 'srcs'
         milestone ordinal: 1
@@ -55,7 +62,7 @@ pipeline {
         script {
           parallel ([
             'build-cpu': { BuildCPU() },
-            'build-gpu-cuda8.0': { BuildCUDA(cuda_version: '8.0') },
+            'build-cpu-rabit-mock': { BuildCPUMock() },
             'build-gpu-cuda9.0': { BuildCUDA(cuda_version: '9.0') },
             'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') },
             'build-gpu-cuda10.1': { BuildCUDA(cuda_version: '10.1') },
@@ -72,7 +79,6 @@ pipeline {
         script {
           parallel ([
             'test-python-cpu': { TestPythonCPU() },
-            'test-python-gpu-cuda8.0': { TestPythonGPU(cuda_version: '8.0') },
             'test-python-gpu-cuda9.0': { TestPythonGPU(cuda_version: '9.0') },
             'test-python-gpu-cuda10.0': { TestPythonGPU(cuda_version: '10.0') },
             'test-python-gpu-cuda10.1': { TestPythonGPU(cuda_version: '10.1') },
@@ -115,7 +121,7 @@ def ClangTidy() {
     def docker_binary = "docker"
     def dockerArgs = "--build-arg CUDA_VERSION=9.2"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${dockerArgs} tests/ci_build/clang_tidy.sh
+    ${dockerRun} ${container_type} ${docker_binary} ${dockerArgs} python3 tests/ci_build/tidy.py
     """
     deleteDir()
   }
@@ -157,7 +163,6 @@ def Doxygen() {
     sh """
     ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/doxygen.sh ${BRANCH_NAME}
     """
-    archiveArtifacts artifacts: "build/${BRANCH_NAME}.tar.bz2", allowEmptyArchive: true
     echo 'Uploading doc...'
     s3Upload file: "build/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "doxygen/${BRANCH_NAME}.tar.bz2"
     deleteDir()
@@ -175,10 +180,10 @@ def BuildCPU() {
     ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """
     // Sanitizer test
-    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 --cap-add SYS_PTRACE'"
+    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 -e UBSAN_OPTIONS=print_stacktrace=1:log_path=ubsan_error.log --cap-add SYS_PTRACE'"
     def docker_args = "--build-arg CMAKE_VERSION=3.12"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address" \
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak;undefined" \
       -DCMAKE_BUILD_TYPE=Debug -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/
     ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """
@@ -186,6 +191,22 @@ def BuildCPU() {
   }
 }
 
+def BuildCPUMock() {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Build CPU with rabit mock"
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_mock_cmake.sh
+    """
+     echo 'Stashing rabit C++ test executable (xgboost)...'
+    stash name: 'xgboost_rabit_tests', includes: 'xgboost'
+    deleteDir()
+  }
+}
+
+
 def BuildCUDA(args) {
   node('linux && cpu') {
     unstash name: 'srcs'
@@ -196,15 +217,14 @@ def BuildCUDA(args) {
     sh """
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux1_x86_64
     """
-    // Stash wheel for CUDA 8.0 / 9.0 target
-    if (args.cuda_version == '8.0') {
-      echo 'Stashing Python wheel...'
-      stash name: 'xgboost_whl_cuda8', includes: 'python-package/dist/*.whl'
-    } else if (args.cuda_version == '9.0') {
+    // Stash wheel for CUDA 9.0 target
+    if (args.cuda_version == '9.0') {
       echo 'Stashing Python wheel...'
       stash name: 'xgboost_whl_cuda9', includes: 'python-package/dist/*.whl'
-      archiveArtifacts artifacts: "python-package/dist/*.whl", allowEmptyArchive: true
+      path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
+      s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
       echo 'Stashing C++ test executable (testxgboost)...'
       stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost'
     }
@@ -238,7 +258,6 @@ def BuildJVMDoc() {
     sh """
     ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME}
     """
-    archiveArtifacts artifacts: "jvm-packages/${BRANCH_NAME}.tar.bz2", allowEmptyArchive: true
     echo 'Uploading doc...'
     s3Upload file: "jvm-packages/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${BRANCH_NAME}.tar.bz2"
     deleteDir()
@@ -254,6 +273,7 @@ def TestPythonCPU() {
     def docker_binary = "docker"
     sh """
     ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/test_python.sh cpu
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/test_python.sh cpu-py35
     """
     deleteDir()
   }
@@ -262,11 +282,7 @@ def TestPythonCPU() {
 def TestPythonGPU(args) {
   nodeReq = (args.multi_gpu) ? 'linux && mgpu' : 'linux && gpu'
   node(nodeReq) {
-    if (args.cuda_version == '8.0') {
-      unstash name: 'xgboost_whl_cuda8'
-    } else {
-      unstash name: 'xgboost_whl_cuda9'
-    }
+    unstash name: 'xgboost_whl_cuda9'
     unstash name: 'srcs'
     echo "Test Python GPU: CUDA ${args.cuda_version}"
     def container_type = "gpu"
@@ -283,6 +299,27 @@ def TestPythonGPU(args) {
       ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh gpu
       """
     }
+    // For CUDA 10.0 target, run cuDF tests too
+    if (args.cuda_version == '10.0') {
+      echo "Running tests with cuDF..."
+      sh """
+      ${dockerRun} cudf ${docker_binary} ${docker_args} tests/ci_build/test_python.sh cudf
+      """
+    }
+    deleteDir()
+  }
+}
+
+def TestCppRabit() {
+  node(nodeReq) {
+    unstash name: 'xgboost_rabit_tests'
+    unstash name: 'srcs'
+    echo "Test C++, rabit mock on"
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/runxgb.sh xgboost tests/ci_build/approx.conf.in
+    """
     deleteDir()
   }
 }
@@ -338,7 +375,7 @@ def TestR(args) {
     def use_r35_flag = (args.use_r35) ? "1" : "0"
     def docker_args = "--build-arg USE_R35=${use_r35_flag}"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_test_rpkg.sh
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_test_rpkg.sh || tests/ci_build/print_r_stacktrace.sh
     """
     deleteDir()
   }

Jenkinsfile-win64

@@ -3,6 +3,11 @@
 
 /* Jenkins pipeline for Windows AMD64 target */
 
+import groovy.transform.Field
+
+@Field
+def commit_id   // necessary to pass a variable from one stage to another
+
 pipeline {
   agent none
   // Build stages
@@ -12,6 +17,7 @@ pipeline {
       steps {
         script {
           checkoutSrcs()
+          commit_id = "${GIT_COMMIT}"
         }
         stash name: 'srcs'
         milestone ordinal: 1
@@ -76,7 +82,7 @@ def BuildWin64() {
     """
     bat """
     cd python-package
-    conda activate && python setup.py bdist_wheel --universal
+    conda activate && python setup.py bdist_wheel --universal && for /R %%i in (dist\\*.whl) DO python ../tests/ci_build/rename_whl.py "%%i" ${commit_id} win_amd64
     """
     echo "Insert vcomp140.dll (OpenMP runtime) into the wheel..."
     bat """
@@ -86,7 +92,8 @@ def BuildWin64() {
     """
     echo 'Stashing Python wheel...'
     stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl'
-    archiveArtifacts artifacts: "python-package/dist/*.whl", allowEmptyArchive: true
+    path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
+    s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
     echo 'Stashing C++ test executable (testxgboost)...'
     stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost.exe'
     deleteDir()

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (c) 2018 by Contributors
+   Copyright (c) 2019 by Contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -42,11 +42,6 @@ ifeq ($(USE_OPENMP), 0)
 endif
 include $(DMLC_CORE)/make/dmlc.mk
 
-# include the plugins
-ifdef XGB_PLUGINS
-include $(XGB_PLUGINS)
-endif
-
 # set compiler defaults for OSX versus *nix
 # let people override either
 OS := $(shell uname)
@@ -67,8 +62,8 @@ export CXX = g++
 endif
 endif
 
-export LDFLAGS= -pthread -lm $(ADD_LDFLAGS) $(DMLC_LDFLAGS) $(PLUGIN_LDFLAGS)
-export CFLAGS= -DDMLC_LOG_CUSTOMIZE=1 -std=c++11 -Wall -Wno-unknown-pragmas -Iinclude $(ADD_CFLAGS) $(PLUGIN_CFLAGS)
+export LDFLAGS= -pthread -lm $(ADD_LDFLAGS) $(DMLC_LDFLAGS)
+export CFLAGS= -DDMLC_LOG_CUSTOMIZE=1 -std=c++11 -Wall -Wno-unknown-pragmas -Iinclude $(ADD_CFLAGS)
 CFLAGS += -I$(DMLC_CORE)/include -I$(RABIT)/include -I$(GTEST_PATH)/include
 #java include path
 export JAVAINCFLAGS = -I${JAVA_HOME}/include -I./java
@@ -130,7 +125,7 @@ $(RABIT)/lib/$(LIB_RABIT): $(wildcard $(RABIT)/src/*.cc)
 jvm: jvm-packages/lib/libxgboost4j.so
 
 SRC = $(wildcard src/*.cc src/*/*.cc)
-ALL_OBJ = $(patsubst src/%.cc, build/%.o, $(SRC)) $(PLUGIN_OBJS)
+ALL_OBJ = $(patsubst src/%.cc, build/%.o, $(SRC))
 AMALGA_OBJ = amalgamation/xgboost-all0.o
 LIB_DEP = $(DMLC_CORE)/libdmlc.a $(RABIT)/lib/$(LIB_RABIT)
 ALL_DEP = $(filter-out build/cli_main.o, $(ALL_OBJ)) $(LIB_DEP)
@@ -142,11 +137,6 @@ build/%.o: src/%.cc
 	$(CXX) $(CFLAGS) -MM -MT build/$*.o $< >build/$*.d
 	$(CXX) -c $(CFLAGS) $< -o $@
 
-build_plugin/%.o: plugin/%.cc
-	@mkdir -p $(@D)
-	$(CXX) $(CFLAGS) -MM -MT build_plugin/$*.o $< >build_plugin/$*.d
-	$(CXX) -c $(CFLAGS) $< -o $@
-
 # The should be equivalent to $(ALL_OBJ)  except for build/cli_main.o
 amalgamation/xgboost-all0.o: amalgamation/xgboost-all0.cc
 	$(CXX) -c $(CFLAGS) $< -o $@
@@ -180,7 +170,7 @@ lint: rcpplint
 	  python-package/xgboost/include python-package/xgboost/lib \
 	  python-package/xgboost/make python-package/xgboost/rabit \
 	  python-package/xgboost/src --pylint-rc ${PWD}/python-package/.pylintrc xgboost \
-	  ${LINT_LANG} include src plugin python-package
+	  ${LINT_LANG} include src python-package
 
 pylint:
 	flake8 --ignore E501 python-package
@@ -200,7 +190,7 @@ cover: check
 endif
 
 clean:
-	$(RM) -rf build build_plugin lib bin *~ */*~ */*/*~ */*/*/*~ */*.o */*/*.o */*/*/*.o #xgboost
+	$(RM) -rf build lib bin *~ */*~ */*/*~ */*/*/*~ */*.o */*/*.o */*/*/*.o #xgboost
 	$(RM) -rf build_tests *.gcov tests/cpp/xgboost_test
 	if [ -d "R-package/src" ]; then \
 		cd R-package/src; \
@@ -231,7 +221,9 @@ pippack: clean_all
 	rm -rf python-package/xgboost/rabit
 	rm -rf python-package/xgboost/src
 	cp -r python-package xgboost-python
-	cp -r Makefile xgboost-python/xgboost/
+	cp -r CMakeLists.txt xgboost-python/xgboost/
+	cp -r cmake xgboost-python/xgboost/
+	cp -r plugin xgboost-python/xgboost/
 	cp -r make xgboost-python/xgboost/
 	cp -r src xgboost-python/xgboost/
 	cp -r tests xgboost-python/xgboost/
@@ -262,10 +254,17 @@ Rpack: clean_all
 	cp -r dmlc-core/include xgboost/src/dmlc-core/include
 	cp -r dmlc-core/src xgboost/src/dmlc-core/src
 	cp ./LICENSE xgboost
-	cat R-package/src/Makevars.in|sed '2s/.*/PKGROOT=./' | sed '3s/.*/ENABLE_STD_THREAD=0/' > xgboost/src/Makevars.in
+	# Modify PKGROOT in Makevars.in
+	cat R-package/src/Makevars.in|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.in
+	# Configure Makevars.win (Windows-specific Makevars, likely using MinGW)
 	cp xgboost/src/Makevars.in xgboost/src/Makevars.win
+	cat xgboost/src/Makevars.in| sed '3s/.*/ENABLE_STD_THREAD=0/' > xgboost/src/Makevars.win
 	sed -i -e 's/@OPENMP_CXXFLAGS@/$$\(SHLIB_OPENMP_CXXFLAGS\)/g' xgboost/src/Makevars.win
 	sed -i -e 's/-pthread/$$\(SHLIB_PTHREAD_FLAGS\)/g' xgboost/src/Makevars.win
+	sed -i -e 's/@ENDIAN_FLAG@/-DDMLC_CMAKE_LITTLE_ENDIAN=1/g' xgboost/src/Makevars.win
+	sed -i -e 's/@BACKTRACE_LIB@//g' xgboost/src/Makevars.win
+	sed -i -e 's/@OPENMP_LIB@//g' xgboost/src/Makevars.win
+	rm -f xgboost/src/Makevars.win-e   # OSX sed create this extra file; remove it
 	bash R-package/remove_warning_suppression_pragma.sh
 	rm xgboost/remove_warning_suppression_pragma.sh
 
@@ -278,4 +277,3 @@ Rcheck: Rbuild
 
 -include build/*.d
 -include build/*/*.d
--include build_plugin/*/*.d

R-package/CMakeLists.txt

@@ -29,6 +29,6 @@ set_target_properties(
   CXX_STANDARD_REQUIRED ON
   POSITION_INDEPENDENT_CODE ON)
 
-set(XGBOOST_DEFINITIONS ${R_DEFINITIONS} PARENT_SCOPE)
+set(XGBOOST_DEFINITIONS "${XGBOOST_DEFINITIONS};${R_DEFINITIONS}" PARENT_SCOPE)
 set(XGBOOST_OBJ_SOURCES $<TARGET_OBJECTS:xgboost-r> PARENT_SCOPE)
 set(LINKED_LIBRARIES_PRIVATE ${LINKED_LIBRARIES_PRIVATE} ${LIBR_CORE_LIBRARY} PARENT_SCOPE)

R-package/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: xgboost
 Type: Package
 Title: Extreme Gradient Boosting
-Version: 0.82.0.1
-Date: 2019-03-11
+Version: 1.0.0.1
+Date: 2019-07-23
 Authors@R: c(
   person("Tianqi", "Chen", role = c("aut"),
          email = "tianqi.tchen@gmail.com"),
@@ -52,7 +52,9 @@ Suggests:
     vcd (>= 1.3),
     testthat,
     lintr,
-    igraph (>= 1.0.1)
+    igraph (>= 1.0.1),
+    jsonlite,
+    float
 Depends:
     R (>= 3.3.0)
 Imports:
@@ -61,5 +63,5 @@ Imports:
     data.table (>= 1.9.6),
     magrittr (>= 1.5),
     stringi (>= 0.5.2)
-RoxygenNote: 6.1.0
+RoxygenNote: 7.0.2
 SystemRequirements: GNU make, C++11

R-package/R/utils.R

@@ -145,7 +145,7 @@ xgb.iter.update <- function(booster_handle, dtrain, iter, obj = NULL) {
   if (is.null(obj)) {
     .Call(XGBoosterUpdateOneIter_R, booster_handle, as.integer(iter), dtrain)
   } else {
-    pred <- predict(booster_handle, dtrain)
+    pred <- predict(booster_handle, dtrain, training = TRUE)
     gpair <- obj(pred, dtrain)
     .Call(XGBoosterBoostOneIter_R, booster_handle, dtrain, gpair$grad, gpair$hess)
   }

R-package/R/xgb.Booster.R

@@ -51,11 +51,13 @@ is.null.handle <- function(handle) {
 # Return a verified to be valid handle out of either xgb.Booster.handle or xgb.Booster
 # internal utility function
 xgb.get.handle <- function(object) {
-  handle <- switch(class(object)[1],
-    xgb.Booster = object$handle,
-    xgb.Booster.handle = object,
+  if (inherits(object, "xgb.Booster")) {
+    handle <- object$handle
+  } else if (inherits(object, "xgb.Booster.handle")) {
+    handle <- object
+  } else {
     stop("argument must be of either xgb.Booster or xgb.Booster.handle class")
-  )
+  }
   if (is.null.handle(handle)) {
     stop("invalid xgb.Booster.handle")
   }
@@ -95,6 +97,7 @@ xgb.get.handle <- function(object) {
 #' saveRDS(bst, "xgb.model.rds")
 #'
 #' bst1 <- readRDS("xgb.model.rds")
+#' if (file.exists("xgb.model.rds")) file.remove("xgb.model.rds")
 #' # the handle is invalid:
 #' print(bst1$handle)
 #'
@@ -136,6 +139,8 @@ xgb.Booster.complete <- function(object, saveraw = TRUE) {
 #' @param reshape whether to reshape the vector of predictions to a matrix form when there are several
 #'        prediction outputs per case. This option has no effect when either of predleaf, predcontrib,
 #'        or predinteraction flags is TRUE.
+#' @param training whether is the prediction result used for training.  For dart booster,
+#'        training predicting will perform dropout.
 #' @param ... Parameters passed to \code{predict.xgb.Booster}
 #'
 #' @details
@@ -285,7 +290,7 @@ xgb.Booster.complete <- function(object, saveraw = TRUE) {
 #' @export
 predict.xgb.Booster <- function(object, newdata, missing = NA, outputmargin = FALSE, ntreelimit = NULL,
                                 predleaf = FALSE, predcontrib = FALSE, approxcontrib = FALSE, predinteraction = FALSE,
-                                reshape = FALSE, ...) {
+                                reshape = FALSE, training = FALSE, ...) {
 
   object <- xgb.Booster.complete(object, saveraw = FALSE)
   if (!inherits(newdata, "xgb.DMatrix"))
@@ -304,7 +309,8 @@ predict.xgb.Booster <- function(object, newdata, missing = NA, outputmargin = FA
   option <- 0L + 1L * as.logical(outputmargin) + 2L * as.logical(predleaf) + 4L * as.logical(predcontrib) +
     8L * as.logical(approxcontrib) + 16L * as.logical(predinteraction)
 
-  ret <- .Call(XGBoosterPredict_R, object$handle, newdata, option[1], as.integer(ntreelimit))
+  ret <- .Call(XGBoosterPredict_R, object$handle, newdata, option[1],
+               as.integer(ntreelimit), as.integer(training))
 
   n_ret <- length(ret)
   n_row <- nrow(newdata)
@@ -418,6 +424,7 @@ predict.xgb.Booster.handle <- function(object, ...) {
 #'
 #' xgb.save(bst, 'xgb.model')
 #' bst1 <- xgb.load('xgb.model')
+#' if (file.exists('xgb.model')) file.remove('xgb.model')
 #' print(xgb.attr(bst1, "my_attribute"))
 #' print(xgb.attributes(bst1))
 #'

R-package/R/xgb.DMatrix.R

@@ -19,6 +19,7 @@
 #' dtrain <- xgb.DMatrix(train$data, label=train$label)
 #' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
 #' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+#' if (file.exists('xgb.DMatrix.data')) file.remove('xgb.DMatrix.data')
 #' @export
 xgb.DMatrix <- function(data, info = list(), missing = NA, silent = FALSE, ...) {
   cnames <- NULL

R-package/R/xgb.DMatrix.save.R

@@ -11,6 +11,7 @@
 #' dtrain <- xgb.DMatrix(train$data, label=train$label)
 #' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
 #' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+#' if (file.exists('xgb.DMatrix.data')) file.remove('xgb.DMatrix.data')
 #' @export
 xgb.DMatrix.save <- function(dmatrix, fname) {
   if (typeof(fname) != "character")

R-package/R/xgb.cv.R

@@ -47,6 +47,8 @@
 #' @param folds \code{list} provides a possibility to use a list of pre-defined CV folds
 #'        (each element must be a vector of test fold's indices). When folds are supplied,
 #'        the \code{nfold} and \code{stratified} parameters are ignored.
+#' @param train_folds \code{list} list specifying which indicies to use for training. If \code{NULL}
+#'        (the default) all indices not specified in \code{folds} will be used for training.
 #' @param verbose \code{boolean}, print the statistics during the process
 #' @param print_every_n Print each n-th iteration evaluation messages when \code{verbose>0}.
 #'        Default is 1 which means all messages are printed. This parameter is passed to the
@@ -114,7 +116,7 @@
 #' @export
 xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL, missing = NA,
                    prediction = FALSE, showsd = TRUE, metrics=list(),
-                   obj = NULL, feval = NULL, stratified = TRUE, folds = NULL,
+                   obj = NULL, feval = NULL, stratified = TRUE, folds = NULL, train_folds = NULL,
                    verbose = TRUE, print_every_n=1L,
                    early_stopping_rounds = NULL, maximize = NULL, callbacks = list(), ...) {
 
@@ -133,8 +135,15 @@ xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL, missing =
 
   # Check the labels
   if ( (inherits(data, 'xgb.DMatrix') && is.null(getinfo(data, 'label'))) ||
-       (!inherits(data, 'xgb.DMatrix') && is.null(label)))
+       (!inherits(data, 'xgb.DMatrix') && is.null(label))) {
     stop("Labels must be provided for CV either through xgb.DMatrix, or through 'label=' when 'data' is matrix")
+  } else if (inherits(data, 'xgb.DMatrix')) {
+    if (!is.null(label))
+      warning("xgb.cv: label will be ignored, since data is of type xgb.DMatrix")
+    cv_label = getinfo(data, 'label')
+  } else {
+    cv_label = label
+  }
 
   # CV folds
   if(!is.null(folds)) {
@@ -144,7 +153,7 @@ xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL, missing =
   } else {
     if (nfold <= 1)
       stop("'nfold' must be > 1")
-    folds <- generate.cv.folds(nfold, nrow(data), stratified, label, params)
+    folds <- generate.cv.folds(nfold, nrow(data), stratified, cv_label, params)
   }
 
   # Potential TODO: sequential CV
@@ -179,10 +188,15 @@ xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL, missing =
 
 
   # create the booster-folds
+  # train_folds
   dall <- xgb.get.DMatrix(data, label, missing)
   bst_folds <- lapply(seq_along(folds), function(k) {
     dtest  <- slice(dall, folds[[k]])
-    dtrain <- slice(dall, unlist(folds[-k]))
+    # code originally contributed by @RolandASc on stackoverflow
+    if(is.null(train_folds))
+       dtrain <- slice(dall, unlist(folds[-k]))
+    else
+       dtrain <- slice(dall, train_folds[[k]])
     handle <- xgb.Booster.handle(params, list(dtrain, dtest))
     list(dtrain = dtrain, bst = handle, watchlist = list(train = dtrain, test=dtest), index = folds[[k]])
   })

R-package/R/xgb.load.R

@@ -28,6 +28,7 @@
 #'                eta = 1, nthread = 2, nrounds = 2,objective = "binary:logistic")
 #' xgb.save(bst, 'xgb.model')
 #' bst <- xgb.load('xgb.model')
+#' if (file.exists('xgb.model')) file.remove('xgb.model')
 #' pred <- predict(bst, test$data)
 #' @export
 xgb.load <- function(modelfile) {

R-package/R/xgb.save.R

@@ -27,6 +27,7 @@
 #'                eta = 1, nthread = 2, nrounds = 2,objective = "binary:logistic")
 #' xgb.save(bst, 'xgb.model')
 #' bst <- xgb.load('xgb.model')
+#' if (file.exists('xgb.model')) file.remove('xgb.model')
 #' pred <- predict(bst, test$data)
 #' @export
 xgb.save <- function(model, fname) {

R-package/R/xgb.train.R

@@ -293,6 +293,9 @@ xgb.train <- function(params = list(), data, nrounds, watchlist = list(),
   }
   # Sort the callbacks into categories
   cb <- categorize.callbacks(callbacks)
+  if (!is.null(params[['seed']])) {
+    warning("xgb.train: `seed` is ignored in R package.  Use `set.seed()` instead.")
+  }
 
   # The tree updating process would need slightly different handling
   is_update <- NVL(params[['process_type']], '.') == 'update'

R-package/R/xgboost.R

@@ -5,8 +5,8 @@
 #' @export
 xgboost <- function(data = NULL, label = NULL, missing = NA, weight = NULL,
                     params = list(), nrounds,
-                    verbose = 1, print_every_n = 1L, 
-                    early_stopping_rounds = NULL, maximize = NULL, 
+                    verbose = 1, print_every_n = 1L,
+                    early_stopping_rounds = NULL, maximize = NULL,
                     save_period = NULL, save_name = "xgboost.model",
                     xgb_model = NULL, callbacks = list(), ...) {
 
@@ -18,16 +18,16 @@ xgboost <- function(data = NULL, label = NULL, missing = NA, weight = NULL,
                    early_stopping_rounds = early_stopping_rounds, maximize = maximize,
                    save_period = save_period, save_name = save_name,
                    xgb_model = xgb_model, callbacks = callbacks, ...)
-  return(bst)
+  return (bst)
 }
 
 #' Training part from Mushroom Data Set
-#' 
+#'
 #' This data set is originally from the Mushroom data set,
 #' UCI Machine Learning Repository.
-#' 
+#'
 #' This data set includes the following fields:
-#' 
+#'
 #' \itemize{
 #'  \item \code{label} the label for each record
 #'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 126 columns.
@@ -35,16 +35,16 @@ xgboost <- function(data = NULL, label = NULL, missing = NA, weight = NULL,
 #'
 #' @references
 #' https://archive.ics.uci.edu/ml/datasets/Mushroom
-#' 
-#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
-#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+#'
+#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
 #' School of Information and Computer Science.
-#' 
+#'
 #' @docType data
 #' @keywords datasets
 #' @name agaricus.train
 #' @usage data(agaricus.train)
-#' @format A list containing a label vector, and a dgCMatrix object with 6513 
+#' @format A list containing a label vector, and a dgCMatrix object with 6513
 #' rows and 127 variables
 NULL
 
@@ -52,9 +52,9 @@ NULL
 #'
 #' This data set is originally from the Mushroom data set,
 #' UCI Machine Learning Repository.
-#' 
+#'
 #' This data set includes the following fields:
-#' 
+#'
 #' \itemize{
 #'  \item \code{label} the label for each record
 #'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 126 columns.
@@ -62,16 +62,16 @@ NULL
 #'
 #' @references
 #' https://archive.ics.uci.edu/ml/datasets/Mushroom
-#' 
-#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
-#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+#'
+#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
 #' School of Information and Computer Science.
-#' 
+#'
 #' @docType data
 #' @keywords datasets
 #' @name agaricus.test
 #' @usage data(agaricus.test)
-#' @format A list containing a label vector, and a dgCMatrix object with 1611 
+#' @format A list containing a label vector, and a dgCMatrix object with 1611
 #' rows and 126 variables
 NULL
 
@@ -107,7 +107,7 @@ NULL
 #' @importFrom graphics par
 #' @importFrom graphics title
 #' @importFrom grDevices rgb
-#' 
+#'
 #' @import methods
 #' @useDynLib xgboost, .registration = TRUE
 NULL

R-package/cleanup

@@ -1,3 +1,4 @@
 #!/bin/sh
 
 rm -f src/Makevars
+rm -f CMakeLists.txt

R-package/configure.ac

@@ -4,6 +4,21 @@ AC_PREREQ(2.62)
 
 AC_INIT([xgboost],[0.6-3],[],[xgboost],[])
 
+# Use this line to set CC variable to a C compiler
+AC_PROG_CC
+
+### Check whether backtrace() is part of libc or the external lib libexecinfo
+AC_MSG_CHECKING([Backtrace lib])
+AC_MSG_RESULT([])
+AC_CHECK_LIB([execinfo], [backtrace], [BACKTRACE_LIB=-lexecinfo], [BACKTRACE_LIB=''])
+
+### Endian detection
+AC_MSG_CHECKING([endian])
+AC_MSG_RESULT([])
+AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include <stdint.h>]], [[const uint16_t endianness = 256; return !!(*(const uint8_t *)&endianness);]])],
+  [ENDIAN_FLAG="-DDMLC_CMAKE_LITTLE_ENDIAN=1"],
+  [ENDIAN_FLAG="-DDMLC_CMAKE_LITTLE_ENDIAN=0"])
+
 OPENMP_CXXFLAGS=""
 
 if test `uname -s` = "Linux"
@@ -13,19 +28,28 @@ fi
 
 if test `uname -s` = "Darwin"
 then
-  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CXXFLAGS)"
+  OPENMP_CXXFLAGS='-Xclang -fopenmp'
+  OPENMP_LIB='/usr/local/lib/libomp.dylib'
   ac_pkg_openmp=no
   AC_MSG_CHECKING([whether OpenMP will work in a package])
-  AC_LANG_CONFTEST(
-  [AC_LANG_PROGRAM([[#include <omp.h>]], [[ return omp_get_num_threads (); ]])])
-  PKG_CFLAGS="${OPENMP_CFLAGS}" PKG_LIBS="${OPENMP_CFLAGS}" "$RBIN" CMD SHLIB conftest.c 1>&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD && "$RBIN" --vanilla -q -e "dyn.load(paste('conftest',.Platform\$dynlib.ext,sep=''))" 1>&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD && ac_pkg_openmp=yes
+  AC_LANG_CONFTEST([AC_LANG_PROGRAM([[#include <omp.h>]], [[ return (omp_get_max_threads() <= 1); ]])])
+  ${CC} -o conftest conftest.c /usr/local/lib/libomp.dylib -Xclang -fopenmp 2>/dev/null && ./conftest && ac_pkg_openmp=yes
   AC_MSG_RESULT([${ac_pkg_openmp}])
   if test "${ac_pkg_openmp}" = no; then
     OPENMP_CXXFLAGS=''
+    OPENMP_LIB=''
+    echo '*****************************************************************************************'
+    echo 'WARNING: OpenMP is unavailable on this Mac OSX system. Training speed may be suboptimal.'
+    echo '         To use all CPU cores for training jobs, you should install OpenMP by running\n'
+    echo '             brew install libomp'
+    echo '*****************************************************************************************'
   fi
 fi
 
 AC_SUBST(OPENMP_CXXFLAGS)
+AC_SUBST(OPENMP_LIB)
+AC_SUBST(ENDIAN_FLAG)
+AC_SUBST(BACKTRACE_LIB)
 AC_CONFIG_FILES([src/Makevars])
 AC_OUTPUT
 

R-package/demo/gpu_accelerated.R

@@ -30,7 +30,7 @@ wl <- list(train = dtrain, test = dtest)
 # - similar to the 'hist'
 # - the fastest option for moderately large datasets
 # - current limitations: max_depth < 16, does not implement guided loss
-# You can use tree_method = 'gpu_exact' for another GPU accelerated algorithm,
+# You can use tree_method = 'gpu_hist' for another GPU accelerated algorithm,
 # which is slower, more memory-hungry, but does not use binning.
 param <- list(objective = 'reg:logistic', eval_metric = 'auc', subsample = 0.5, nthread = 4,
               max_bin = 64, tree_method = 'gpu_hist')

R-package/man/agaricus.test.Rd

@@ -4,7 +4,7 @@
 \name{agaricus.test}
 \alias{agaricus.test}
 \title{Test part from Mushroom Data Set}
-\format{A list containing a label vector, and a dgCMatrix object with 1611 
+\format{A list containing a label vector, and a dgCMatrix object with 1611
 rows and 126 variables}
 \usage{
 data(agaricus.test)
@@ -24,8 +24,8 @@ This data set includes the following fields:
 \references{
 https://archive.ics.uci.edu/ml/datasets/Mushroom
 
-Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
-[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
 School of Information and Computer Science.
 }
 \keyword{datasets}

R-package/man/agaricus.train.Rd

@@ -4,7 +4,7 @@
 \name{agaricus.train}
 \alias{agaricus.train}
 \title{Training part from Mushroom Data Set}
-\format{A list containing a label vector, and a dgCMatrix object with 6513 
+\format{A list containing a label vector, and a dgCMatrix object with 6513
 rows and 127 variables}
 \usage{
 data(agaricus.train)
@@ -24,8 +24,8 @@ This data set includes the following fields:
 \references{
 https://archive.ics.uci.edu/ml/datasets/Mushroom
 
-Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
-[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
 School of Information and Computer Science.
 }
 \keyword{datasets}

R-package/man/cb.early.stop.Rd

@@ -4,8 +4,12 @@
 \alias{cb.early.stop}
 \title{Callback closure to activate the early stopping.}
 \usage{
-cb.early.stop(stopping_rounds, maximize = FALSE, metric_name = NULL,
-  verbose = TRUE)
+cb.early.stop(
+  stopping_rounds,
+  maximize = FALSE,
+  metric_name = NULL,
+  verbose = TRUE
+)
 }
 \arguments{
 \item{stopping_rounds}{The number of rounds with no improvement in

R-package/man/predict.xgb.Booster.Rd

@@ -5,10 +5,20 @@
 \alias{predict.xgb.Booster.handle}
 \title{Predict method for eXtreme Gradient Boosting model}
 \usage{
-\method{predict}{xgb.Booster}(object, newdata, missing = NA,
-  outputmargin = FALSE, ntreelimit = NULL, predleaf = FALSE,
-  predcontrib = FALSE, approxcontrib = FALSE,
-  predinteraction = FALSE, reshape = FALSE, ...)
+\method{predict}{xgb.Booster}(
+  object,
+  newdata,
+  missing = NA,
+  outputmargin = FALSE,
+  ntreelimit = NULL,
+  predleaf = FALSE,
+  predcontrib = FALSE,
+  approxcontrib = FALSE,
+  predinteraction = FALSE,
+  reshape = FALSE,
+  training = FALSE,
+  ...
+)
 
 \method{predict}{xgb.Booster.handle}(object, ...)
 }
@@ -39,6 +49,9 @@ It will use all the trees by default (\code{NULL} value).}
 prediction outputs per case. This option has no effect when either of predleaf, predcontrib,
 or predinteraction flags is TRUE.}
 
+\item{training}{whether is the prediction result used for training.  For dart booster,
+training predicting will perform dropout.}
+
 \item{...}{Parameters passed to \code{predict.xgb.Booster}}
 }
 \value{

R-package/man/xgb.Booster.complete.Rd

@@ -39,6 +39,7 @@ bst <- xgboost(data = agaricus.train$data, label = agaricus.train$label, max_dep
 saveRDS(bst, "xgb.model.rds")
 
 bst1 <- readRDS("xgb.model.rds")
+if (file.exists("xgb.model.rds")) file.remove("xgb.model.rds")
 # the handle is invalid:
 print(bst1$handle)
 

R-package/man/xgb.DMatrix.Rd

@@ -31,4 +31,5 @@ train <- agaricus.train
 dtrain <- xgb.DMatrix(train$data, label=train$label)
 xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
 dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+if (file.exists('xgb.DMatrix.data')) file.remove('xgb.DMatrix.data')
 }

R-package/man/xgb.DMatrix.save.Rd

@@ -20,4 +20,5 @@ train <- agaricus.train
 dtrain <- xgb.DMatrix(train$data, label=train$label)
 xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
 dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+if (file.exists('xgb.DMatrix.data')) file.remove('xgb.DMatrix.data')
 }

R-package/man/xgb.attr.Rd

@@ -73,6 +73,7 @@ xgb.attributes(bst) <- list(a = 123, b = "abc")
 
 xgb.save(bst, 'xgb.model')
 bst1 <- xgb.load('xgb.model')
+if (file.exists('xgb.model')) file.remove('xgb.model')
 print(xgb.attr(bst1, "my_attribute"))
 print(xgb.attributes(bst1))
 

R-package/man/xgb.create.features.Rd

@@ -87,6 +87,6 @@ accuracy.after <- sum((predict(bst, new.dtest) >= 0.5) == agaricus.test$label) /
 
 # Here the accuracy was already good and is now perfect.
 cat(paste("The accuracy was", accuracy.before, "before adding leaf features and it is now",
-          accuracy.after, "!\\n"))
+          accuracy.after, "!\n"))
 
 }

R-package/man/xgb.cv.Rd

@@ -4,12 +4,28 @@
 \alias{xgb.cv}
 \title{Cross Validation}
 \usage{
-xgb.cv(params = list(), data, nrounds, nfold, label = NULL,
-  missing = NA, prediction = FALSE, showsd = TRUE,
-  metrics = list(), obj = NULL, feval = NULL, stratified = TRUE,
-  folds = NULL, verbose = TRUE, print_every_n = 1L,
-  early_stopping_rounds = NULL, maximize = NULL, callbacks = list(),
-  ...)
+xgb.cv(
+  params = list(),
+  data,
+  nrounds,
+  nfold,
+  label = NULL,
+  missing = NA,
+  prediction = FALSE,
+  showsd = TRUE,
+  metrics = list(),
+  obj = NULL,
+  feval = NULL,
+  stratified = TRUE,
+  folds = NULL,
+  train_folds = NULL,
+  verbose = TRUE,
+  print_every_n = 1L,
+  early_stopping_rounds = NULL,
+  maximize = NULL,
+  callbacks = list(),
+  ...
+)
 }
 \arguments{
 \item{params}{the list of parameters. Commonly used ones are:
@@ -70,6 +86,9 @@ by the values of outcome labels.}
 (each element must be a vector of test fold's indices). When folds are supplied,
 the \code{nfold} and \code{stratified} parameters are ignored.}
 
+\item{train_folds}{\code{list} list specifying which indicies to use for training. If \code{NULL}
+(the default) all indices not specified in \code{folds} will be used for training.}
+
 \item{verbose}{\code{boolean}, print the statistics during the process}
 
 \item{print_every_n}{Print each n-th iteration evaluation messages when \code{verbose>0}.

R-package/man/xgb.dump.Rd

@@ -4,8 +4,14 @@
 \alias{xgb.dump}
 \title{Dump an xgboost model in text format.}
 \usage{
-xgb.dump(model, fname = NULL, fmap = "", with_stats = FALSE,
-  dump_format = c("text", "json"), ...)
+xgb.dump(
+  model,
+  fname = NULL,
+  fmap = "",
+  with_stats = FALSE,
+  dump_format = c("text", "json"),
+  ...
+)
 }
 \arguments{
 \item{model}{the model object.}

R-package/man/xgb.importance.Rd

@@ -4,8 +4,14 @@
 \alias{xgb.importance}
 \title{Importance of features in a model.}
 \usage{
-xgb.importance(feature_names = NULL, model = NULL, trees = NULL,
-  data = NULL, label = NULL, target = NULL)
+xgb.importance(
+  feature_names = NULL,
+  model = NULL,
+  trees = NULL,
+  data = NULL,
+  label = NULL,
+  target = NULL
+)
 }
 \arguments{
 \item{feature_names}{character vector of feature names. If the model already

R-package/man/xgb.load.Rd

@@ -33,6 +33,7 @@ bst <- xgboost(data = train$data, label = train$label, max_depth = 2,
                eta = 1, nthread = 2, nrounds = 2,objective = "binary:logistic")
 xgb.save(bst, 'xgb.model')
 bst <- xgb.load('xgb.model')
+if (file.exists('xgb.model')) file.remove('xgb.model')
 pred <- predict(bst, test$data)
 }
 \seealso{

R-package/man/xgb.model.dt.tree.Rd

@@ -4,8 +4,14 @@
 \alias{xgb.model.dt.tree}
 \title{Parse a boosted tree model text dump}
 \usage{
-xgb.model.dt.tree(feature_names = NULL, model = NULL, text = NULL,
-  trees = NULL, use_int_id = FALSE, ...)
+xgb.model.dt.tree(
+  feature_names = NULL,
+  model = NULL,
+  text = NULL,
+  trees = NULL,
+  use_int_id = FALSE,
+  ...
+)
 }
 \arguments{
 \item{feature_names}{character vector of feature names. If the model already

R-package/man/xgb.plot.deepness.Rd

@@ -5,11 +5,17 @@
 \alias{xgb.plot.deepness}
 \title{Plot model trees deepness}
 \usage{
-xgb.ggplot.deepness(model = NULL, which = c("2x1", "max.depth",
-  "med.depth", "med.weight"))
+xgb.ggplot.deepness(
+  model = NULL,
+  which = c("2x1", "max.depth", "med.depth", "med.weight")
+)
 
-xgb.plot.deepness(model = NULL, which = c("2x1", "max.depth",
-  "med.depth", "med.weight"), plot = TRUE, ...)
+xgb.plot.deepness(
+  model = NULL,
+  which = c("2x1", "max.depth", "med.depth", "med.weight"),
+  plot = TRUE,
+  ...
+)
 }
 \arguments{
 \item{model}{either an \code{xgb.Booster} model generated by the \code{xgb.train} function

R-package/man/xgb.plot.importance.Rd

@@ -5,12 +5,25 @@
 \alias{xgb.plot.importance}
 \title{Plot feature importance as a bar graph}
 \usage{
-xgb.ggplot.importance(importance_matrix = NULL, top_n = NULL,
-  measure = NULL, rel_to_first = FALSE, n_clusters = c(1:10), ...)
+xgb.ggplot.importance(
+  importance_matrix = NULL,
+  top_n = NULL,
+  measure = NULL,
+  rel_to_first = FALSE,
+  n_clusters = c(1:10),
+  ...
+)
 
-xgb.plot.importance(importance_matrix = NULL, top_n = NULL,
-  measure = NULL, rel_to_first = FALSE, left_margin = 10,
-  cex = NULL, plot = TRUE, ...)
+xgb.plot.importance(
+  importance_matrix = NULL,
+  top_n = NULL,
+  measure = NULL,
+  rel_to_first = FALSE,
+  left_margin = 10,
+  cex = NULL,
+  plot = TRUE,
+  ...
+)
 }
 \arguments{
 \item{importance_matrix}{a \code{data.table} returned by \code{\link{xgb.importance}}.}

R-package/man/xgb.plot.multi.trees.Rd

@@ -4,8 +4,15 @@
 \alias{xgb.plot.multi.trees}
 \title{Project all trees on one tree and plot it}
 \usage{
-xgb.plot.multi.trees(model, feature_names = NULL, features_keep = 5,
-  plot_width = NULL, plot_height = NULL, render = TRUE, ...)
+xgb.plot.multi.trees(
+  model,
+  feature_names = NULL,
+  features_keep = 5,
+  plot_width = NULL,
+  plot_height = NULL,
+  render = TRUE,
+  ...
+)
 }
 \arguments{
 \item{model}{produced by the \code{xgb.train} function.}

R-package/man/xgb.plot.shap.Rd

@@ -4,13 +4,33 @@
 \alias{xgb.plot.shap}
 \title{SHAP contribution dependency plots}
 \usage{
-xgb.plot.shap(data, shap_contrib = NULL, features = NULL, top_n = 1,
-  model = NULL, trees = NULL, target_class = NULL,
-  approxcontrib = FALSE, subsample = NULL, n_col = 1, col = rgb(0,
-  0, 1, 0.2), pch = ".", discrete_n_uniq = 5, discrete_jitter = 0.01,
-  ylab = "SHAP", plot_NA = TRUE, col_NA = rgb(0.7, 0, 1, 0.6),
-  pch_NA = ".", pos_NA = 1.07, plot_loess = TRUE, col_loess = 2,
-  span_loess = 0.5, which = c("1d", "2d"), plot = TRUE, ...)
+xgb.plot.shap(
+  data,
+  shap_contrib = NULL,
+  features = NULL,
+  top_n = 1,
+  model = NULL,
+  trees = NULL,
+  target_class = NULL,
+  approxcontrib = FALSE,
+  subsample = NULL,
+  n_col = 1,
+  col = rgb(0, 0, 1, 0.2),
+  pch = ".",
+  discrete_n_uniq = 5,
+  discrete_jitter = 0.01,
+  ylab = "SHAP",
+  plot_NA = TRUE,
+  col_NA = rgb(0.7, 0, 1, 0.6),
+  pch_NA = ".",
+  pos_NA = 1.07,
+  plot_loess = TRUE,
+  col_loess = 2,
+  span_loess = 0.5,
+  which = c("1d", "2d"),
+  plot = TRUE,
+  ...
+)
 }
 \arguments{
 \item{data}{data as a \code{matrix} or \code{dgCMatrix}.}

R-package/man/xgb.plot.tree.Rd

@@ -4,9 +4,16 @@
 \alias{xgb.plot.tree}
 \title{Plot a boosted tree model}
 \usage{
-xgb.plot.tree(feature_names = NULL, model = NULL, trees = NULL,
-  plot_width = NULL, plot_height = NULL, render = TRUE,
-  show_node_id = FALSE, ...)
+xgb.plot.tree(
+  feature_names = NULL,
+  model = NULL,
+  trees = NULL,
+  plot_width = NULL,
+  plot_height = NULL,
+  render = TRUE,
+  show_node_id = FALSE,
+  ...
+)
 }
 \arguments{
 \item{feature_names}{names of each feature as a \code{character} vector.}

R-package/man/xgb.save.Rd

@@ -33,6 +33,7 @@ bst <- xgboost(data = train$data, label = train$label, max_depth = 2,
                eta = 1, nthread = 2, nrounds = 2,objective = "binary:logistic")
 xgb.save(bst, 'xgb.model')
 bst <- xgb.load('xgb.model')
+if (file.exists('xgb.model')) file.remove('xgb.model')
 pred <- predict(bst, test$data)
 }
 \seealso{

R-package/man/xgb.train.Rd

@@ -5,17 +5,41 @@
 \alias{xgboost}
 \title{eXtreme Gradient Boosting Training}
 \usage{
-xgb.train(params = list(), data, nrounds, watchlist = list(),
-  obj = NULL, feval = NULL, verbose = 1, print_every_n = 1L,
-  early_stopping_rounds = NULL, maximize = NULL, save_period = NULL,
-  save_name = "xgboost.model", xgb_model = NULL, callbacks = list(),
-  ...)
+xgb.train(
+  params = list(),
+  data,
+  nrounds,
+  watchlist = list(),
+  obj = NULL,
+  feval = NULL,
+  verbose = 1,
+  print_every_n = 1L,
+  early_stopping_rounds = NULL,
+  maximize = NULL,
+  save_period = NULL,
+  save_name = "xgboost.model",
+  xgb_model = NULL,
+  callbacks = list(),
+  ...
+)
 
-xgboost(data = NULL, label = NULL, missing = NA, weight = NULL,
-  params = list(), nrounds, verbose = 1, print_every_n = 1L,
-  early_stopping_rounds = NULL, maximize = NULL, save_period = NULL,
-  save_name = "xgboost.model", xgb_model = NULL, callbacks = list(),
-  ...)
+xgboost(
+  data = NULL,
+  label = NULL,
+  missing = NA,
+  weight = NULL,
+  params = list(),
+  nrounds,
+  verbose = 1,
+  print_every_n = 1L,
+  early_stopping_rounds = NULL,
+  maximize = NULL,
+  save_period = NULL,
+  save_name = "xgboost.model",
+  xgb_model = NULL,
+  callbacks = list(),
+  ...
+)
 }
 \arguments{
 \item{params}{the list of parameters.

R-package/src/Makevars.in

@@ -17,8 +17,8 @@ endif
 $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
-PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ -pthread
-PKG_LIBS = @OPENMP_CXXFLAGS@ -pthread
+PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ @ENDIAN_FLAG@ -pthread
+PKG_LIBS = @OPENMP_CXXFLAGS@ @OPENMP_LIB@ @ENDIAN_FLAG@ @BACKTRACE_LIB@ -pthread
 OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o\
          $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o\
          $(PKGROOT)/rabit/src/engine_empty.o $(PKGROOT)/rabit/src/c_api.o

R-package/src/init.c

@@ -24,7 +24,7 @@ extern SEXP XGBoosterGetAttr_R(SEXP, SEXP);
 extern SEXP XGBoosterLoadModelFromRaw_R(SEXP, SEXP);
 extern SEXP XGBoosterLoadModel_R(SEXP, SEXP);
 extern SEXP XGBoosterModelToRaw_R(SEXP);
-extern SEXP XGBoosterPredict_R(SEXP, SEXP, SEXP, SEXP);
+extern SEXP XGBoosterPredict_R(SEXP, SEXP, SEXP, SEXP, SEXP);
 extern SEXP XGBoosterSaveModel_R(SEXP, SEXP);
 extern SEXP XGBoosterSetAttr_R(SEXP, SEXP, SEXP);
 extern SEXP XGBoosterSetParam_R(SEXP, SEXP, SEXP);
@@ -50,7 +50,7 @@ static const R_CallMethodDef CallEntries[] = {
   {"XGBoosterLoadModelFromRaw_R", (DL_FUNC) &XGBoosterLoadModelFromRaw_R, 2},
   {"XGBoosterLoadModel_R",        (DL_FUNC) &XGBoosterLoadModel_R,        2},
   {"XGBoosterModelToRaw_R",       (DL_FUNC) &XGBoosterModelToRaw_R,       1},
-  {"XGBoosterPredict_R",          (DL_FUNC) &XGBoosterPredict_R,          4},
+  {"XGBoosterPredict_R",          (DL_FUNC) &XGBoosterPredict_R,          5},
   {"XGBoosterSaveModel_R",        (DL_FUNC) &XGBoosterSaveModel_R,        2},
   {"XGBoosterSetAttr_R",          (DL_FUNC) &XGBoosterSetAttr_R,          3},
   {"XGBoosterSetParam_R",         (DL_FUNC) &XGBoosterSetParam_R,         3},

R-package/src/xgboost_R.cc

@@ -136,9 +136,10 @@ SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset) {
     idxvec[i] = INTEGER(idxset)[i] - 1;
   }
   DMatrixHandle res;
-  CHECK_CALL(XGDMatrixSliceDMatrix(R_ExternalPtrAddr(handle),
-                                   BeginPtr(idxvec), len,
-                                   &res));
+  CHECK_CALL(XGDMatrixSliceDMatrixEx(R_ExternalPtrAddr(handle),
+                                     BeginPtr(idxvec), len,
+                                     &res,
+                                     0));
   ret = PROTECT(R_MakeExternalPtr(res, R_NilValue, R_NilValue));
   R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
   R_API_END();
@@ -165,7 +166,9 @@ SEXP XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array) {
     for (int i = 0; i < len; ++i) {
       vec[i] = static_cast<unsigned>(INTEGER(array)[i]);
     }
-    CHECK_CALL(XGDMatrixSetGroup(R_ExternalPtrAddr(handle), BeginPtr(vec), len));
+    CHECK_CALL(XGDMatrixSetUIntInfo(R_ExternalPtrAddr(handle),
+                                    CHAR(asChar(field)),
+                                    BeginPtr(vec), len));
   } else {
     std::vector<float> vec(len);
     #pragma omp parallel for schedule(static)
@@ -173,8 +176,8 @@ SEXP XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array) {
       vec[i] = REAL(array)[i];
     }
     CHECK_CALL(XGDMatrixSetFloatInfo(R_ExternalPtrAddr(handle),
-                                   CHAR(asChar(field)),
-                                   BeginPtr(vec), len));
+                                     CHAR(asChar(field)),
+                                     BeginPtr(vec), len));
   }
   R_API_END();
   return R_NilValue;
@@ -292,24 +295,26 @@ SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames) {
     vec_sptr.push_back(vec_names[i].c_str());
   }
   CHECK_CALL(XGBoosterEvalOneIter(R_ExternalPtrAddr(handle),
-                                asInteger(iter),
-                                BeginPtr(vec_dmats),
-                                BeginPtr(vec_sptr),
-                                len, &ret));
+                                  asInteger(iter),
+                                  BeginPtr(vec_dmats),
+                                  BeginPtr(vec_sptr),
+                                  len, &ret));
   R_API_END();
   return mkString(ret);
 }
 
-SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP option_mask, SEXP ntree_limit) {
+SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP option_mask,
+                        SEXP ntree_limit, SEXP training) {
   SEXP ret;
   R_API_BEGIN();
   bst_ulong olen;
   const float *res;
   CHECK_CALL(XGBoosterPredict(R_ExternalPtrAddr(handle),
-                            R_ExternalPtrAddr(dmat),
-                            asInteger(option_mask),
-                            asInteger(ntree_limit),
-                            &olen, &res));
+                              R_ExternalPtrAddr(dmat),
+                              asInteger(option_mask),
+                              asInteger(ntree_limit),
+                              asInteger(training),
+                              &olen, &res));
   ret = PROTECT(allocVector(REALSXP, olen));
   for (size_t i = 0; i < olen; ++i) {
     REAL(ret)[i] = res[i];

R-package/src/xgboost_R.h

@@ -148,8 +148,10 @@ XGB_DLL SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evn
  * \param dmat data matrix
  * \param option_mask output_margin:1 predict_leaf:2
  * \param ntree_limit limit number of trees used in prediction
+ * \param training Whether the prediction value is used for training.
  */
-XGB_DLL SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP option_mask, SEXP ntree_limit);
+XGB_DLL SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP option_mask,
+                                SEXP ntree_limit, SEXP training);
 /*!
  * \brief load model from existing file
  * \param handle handle

R-package/tests/testthat/test_basic.R

@@ -27,7 +27,7 @@ test_that("train and predict binary classification", {
 
   pred <- predict(bst, test$data)
   expect_length(pred, 1611)
-  
+
   pred1 <- predict(bst, train$data, ntreelimit = 1)
   expect_length(pred1, 6513)
   err_pred1 <- sum((pred1 > 0.5) != train$label)/length(train$label)
@@ -35,6 +35,54 @@ test_that("train and predict binary classification", {
   expect_lt(abs(err_pred1 - err_log), 10e-6)
 })
 
+test_that("dart prediction works", {
+  nrounds = 32
+  set.seed(1994)
+
+  d <- cbind(
+    x1 = rnorm(100),
+    x2 = rnorm(100),
+    x3 = rnorm(100))
+  y <- d[,"x1"] + d[,"x2"]^2 +
+    ifelse(d[,"x3"] > .5, d[,"x3"]^2, 2^d[,"x3"]) +
+    rnorm(100)
+
+  set.seed(1994)
+  booster_by_xgboost <- xgboost(data = d, label = y, max_depth = 2, booster = "dart",
+                                rate_drop = 0.5, one_drop = TRUE,
+                                eta = 1, nthread = 2, nrounds = nrounds, objective = "reg:squarederror")
+  pred_by_xgboost_0 <- predict(booster_by_xgboost, newdata = d, ntreelimit = 0)
+  pred_by_xgboost_1 <- predict(booster_by_xgboost, newdata = d, ntreelimit = nrounds)
+  expect_true(all(matrix(pred_by_xgboost_0, byrow=TRUE) == matrix(pred_by_xgboost_1, byrow=TRUE)))
+
+  pred_by_xgboost_2 <- predict(booster_by_xgboost, newdata = d, training = TRUE)
+  expect_false(all(matrix(pred_by_xgboost_0, byrow=TRUE) == matrix(pred_by_xgboost_2, byrow=TRUE)))
+
+  set.seed(1994)
+  dtrain <- xgb.DMatrix(data=d, info = list(label=y))
+  booster_by_train <- xgb.train( params = list(
+                                   booster = "dart",
+                                   max_depth = 2,
+                                   eta = 1,
+                                   rate_drop = 0.5,
+                                   one_drop = TRUE,
+                                   nthread = 1,
+                                   tree_method= "exact",
+                                   verbosity = 3,
+                                   objective = "reg:squarederror"
+                                 ),
+                                data = dtrain,
+                                nrounds = nrounds
+                                )
+  pred_by_train_0 <- predict(booster_by_train, newdata = dtrain, ntreelimit = 0)
+  pred_by_train_1 <- predict(booster_by_train, newdata = dtrain, ntreelimit = nrounds)
+  pred_by_train_2 <- predict(booster_by_train, newdata = dtrain, training = TRUE)
+
+  expect_true(all(matrix(pred_by_train_0, byrow=TRUE) == matrix(pred_by_xgboost_0, byrow=TRUE)))
+  expect_true(all(matrix(pred_by_train_1, byrow=TRUE) == matrix(pred_by_xgboost_1, byrow=TRUE)))
+  expect_true(all(matrix(pred_by_train_2, byrow=TRUE) == matrix(pred_by_xgboost_2, byrow=TRUE)))
+})
+
 test_that("train and predict softprob", {
   lb <- as.numeric(iris$Species) - 1
   set.seed(11)
@@ -74,7 +122,7 @@ test_that("train and predict softmax", {
   expect_false(is.null(bst$evaluation_log))
   expect_lt(bst$evaluation_log[, min(train_merror)], 0.025)
   expect_equal(bst$niter * 3, xgb.ntree(bst))
-  
+
   pred <- predict(bst, as.matrix(iris[, -5]))
   expect_length(pred, nrow(iris))
   err <- sum(pred != lb)/length(lb)
@@ -90,12 +138,12 @@ test_that("train and predict RF", {
                  num_parallel_tree = 20, subsample = 0.6, colsample_bytree = 0.1)
   expect_equal(bst$niter, 1)
   expect_equal(xgb.ntree(bst), 20)
-  
+
   pred <- predict(bst, train$data)
   pred_err <- sum((pred > 0.5) != lb)/length(lb)
   expect_lt(abs(bst$evaluation_log[1, train_error] - pred_err), 10e-6)
   #expect_lt(pred_err, 0.03)
-  
+
   pred <- predict(bst, train$data, ntreelimit = 20)
   pred_err_20 <- sum((pred > 0.5) != lb)/length(lb)
   expect_equal(pred_err_20, pred_err)
@@ -191,13 +239,27 @@ test_that("xgb.cv works", {
   expect_false(is.null(cv$call))
 })
 
+test_that("xgb.cv works with stratified folds", {
+  dtrain <- xgb.DMatrix(train$data, label = train$label)
+  set.seed(314159)
+  cv <- xgb.cv(data = dtrain, max_depth = 2, nfold = 5,
+               eta = 1., nthread = 2, nrounds = 2, objective = "binary:logistic",
+               verbose=TRUE, stratified = FALSE)
+  set.seed(314159)
+  cv2 <- xgb.cv(data = dtrain, max_depth = 2, nfold = 5,
+                eta = 1., nthread = 2, nrounds = 2, objective = "binary:logistic",
+                verbose=TRUE, stratified = TRUE)
+  # Stratified folds should result in a different evaluation logs
+  expect_true(all(cv$evaluation_log[, test_error_mean] != cv2$evaluation_log[, test_error_mean]))
+})
+
 test_that("train and predict with non-strict classes", {
   # standard dense matrix input
   train_dense <- as.matrix(train$data)
   bst <- xgboost(data = train_dense, label = train$label, max_depth = 2,
                  eta = 1, nthread = 2, nrounds = 2, objective = "binary:logistic", verbose = 0)
   pr0 <- predict(bst, train_dense)
-  
+
   # dense matrix-like input of non-matrix class
   class(train_dense) <- 'shmatrix'
   expect_true(is.matrix(train_dense))
@@ -207,7 +269,7 @@ test_that("train and predict with non-strict classes", {
     , regexp = NA)
   expect_error(pr <- predict(bst, train_dense), regexp = NA)
   expect_equal(pr0, pr)
-  
+
   # dense matrix-like input of non-matrix class with some inheritance
   class(train_dense) <- c('pphmatrix','shmatrix')
   expect_true(is.matrix(train_dense))
@@ -217,7 +279,7 @@ test_that("train and predict with non-strict classes", {
     , regexp = NA)
   expect_error(pr <- predict(bst, train_dense), regexp = NA)
   expect_equal(pr0, pr)
-  
+
   # when someone inhertis from xgb.Booster, it should still be possible to use it as xgb.Booster
   class(bst) <- c('super.Booster', 'xgb.Booster')
   expect_error(pr <- predict(bst, train_dense), regexp = NA)

R-package/tests/testthat/test_callbacks.R

@@ -236,7 +236,7 @@ test_that("early stopping using a specific metric works", {
   expect_equal(length(pred), 1611)
   logloss_pred <- sum(-ltest * log(pred) - (1 - ltest) * log(1 - pred)) / length(ltest)
   logloss_log <- bst$evaluation_log[bst$best_iteration, test_logloss]
-  expect_equal(logloss_log, logloss_pred, tolerance = 5e-6)
+  expect_equal(logloss_log, logloss_pred, tolerance = 1e-5)
 })
 
 test_that("early stopping xgb.cv works", {
@@ -285,7 +285,8 @@ test_that("prediction in early-stopping xgb.cv works", {
   set.seed(11)
   expect_output(
     cv <- xgb.cv(param, dtrain, nfold = 5, eta = 0.1, nrounds = 20,
-                 early_stopping_rounds = 5, maximize = FALSE, prediction = TRUE)
+                 early_stopping_rounds = 5, maximize = FALSE, stratified = FALSE,
+                 prediction = TRUE)
   , "Stopping. Best iteration")
   
   expect_false(is.null(cv$best_iteration))

R-package/tests/testthat/test_custom_objective.R

@@ -31,7 +31,6 @@ num_round <- 2
 test_that("custom objective works", {
   bst <- xgb.train(param, dtrain, num_round, watchlist)
   expect_equal(class(bst), "xgb.Booster")
-  expect_equal(length(bst$raw), 1100)
   expect_false(is.null(bst$evaluation_log))
   expect_false(is.null(bst$evaluation_log$eval_error))
   expect_lt(bst$evaluation_log[num_round, eval_error], 0.03)
@@ -58,5 +57,4 @@ test_that("custom objective using DMatrix attr works", {
   param$objective = logregobjattr
   bst <- xgb.train(param, dtrain, num_round, watchlist)
   expect_equal(class(bst), "xgb.Booster")
-  expect_equal(length(bst$raw), 1100)
 })

R-package/tests/testthat/test_helpers.R

@@ -142,6 +142,44 @@ test_that("predict feature contributions works", {
   }
 })
 
+test_that("SHAPs sum to predictions, with or without DART", {
+  d <- cbind(
+    x1 = rnorm(100),
+    x2 = rnorm(100),
+    x3 = rnorm(100))
+  y <- d[,"x1"] + d[,"x2"]^2 +
+    ifelse(d[,"x3"] > .5, d[,"x3"]^2, 2^d[,"x3"]) +
+    rnorm(100)
+  nrounds <- 30
+
+  for (booster in list("gbtree", "dart")) {
+    fit <- xgboost(
+      params = c(
+        list(
+          booster = booster,
+          objective = "reg:squarederror",
+          eval_metric = "rmse"),
+        if (booster == "dart")
+          list(rate_drop = .01, one_drop = T)),
+      data = d,
+      label = y,
+      nrounds = nrounds)
+
+    pr <- function(...)
+      predict(fit, newdata = d, ...)
+    pred <- pr()
+    shap <- pr(predcontrib = T)
+    shapi <- pr(predinteraction = T)
+    tol = 1e-5
+
+    expect_equal(rowSums(shap), pred, tol = tol)
+    expect_equal(apply(shapi, 1, sum), pred, tol = tol)
+    for (i in 1 : nrow(d))
+      for (f in list(rowSums, colSums))
+        expect_equal(f(shapi[i,,]), shap[i,], tol = tol)
+  }
+})
+
 test_that("xgb-attribute functionality", {
   val <- "my attribute value"
   list.val <- list(my_attr=val, a=123, b='ok')
@@ -163,6 +201,7 @@ test_that("xgb-attribute functionality", {
   # serializing:
   xgb.save(bst.Tree, 'xgb.model')
   bst <- xgb.load('xgb.model')
+  if (file.exists('xgb.model')) file.remove('xgb.model')
   expect_equal(xgb.attr(bst, "my_attr"), val)
   expect_equal(xgb.attributes(bst), list.ch)
   # deletion:
@@ -199,10 +238,12 @@ if (grepl('Windows', Sys.info()[['sysname']]) ||
 test_that("xgb.Booster serializing as R object works", {
   saveRDS(bst.Tree, 'xgb.model.rds')
   bst <- readRDS('xgb.model.rds')
+  if (file.exists('xgb.model.rds')) file.remove('xgb.model.rds')
   dtrain <- xgb.DMatrix(sparse_matrix, label = label)
   expect_equal(predict(bst.Tree, dtrain), predict(bst, dtrain), tolerance = float_tolerance)
   expect_equal(xgb.dump(bst.Tree), xgb.dump(bst))
   xgb.save(bst, 'xgb.model')
+  if (file.exists('xgb.model')) file.remove('xgb.model')
   nil_ptr <- new("externalptr")
   class(nil_ptr) <- "xgb.Booster.handle"
   expect_true(identical(bst$handle, nil_ptr))

R-package/vignettes/xgboostfromJSON.Rmd

@@ -70,7 +70,7 @@ First let's dump the model to JSON:
 
 ```{r}
 bst_json <- xgb.dump(bst, with_stats = FALSE, dump_format='json')
-bst_from_json <- jsonlite::fromJSON(bst_json, simplifyDataFrame = FALSE)
+bst_from_json <- fromJSON(bst_json, simplifyDataFrame = FALSE)
 node <- bst_from_json[[1]]
 cat(bst_json)
 ```
@@ -186,4 +186,4 @@ bst_from_json_preds <- ifelse(fl(data$dates)<fl(node$split_condition),
 bst_preds == bst_from_json_preds
 ```
 
-All equal.  What's the lesson?  We have to ensure that all calculations are done with 32-bit floating point operators if we want to reproduce the results that we see with xgboost.
+All equal.  What's the lesson?  We have to ensure that all calculations are done with 32-bit floating point operators if we want to reproduce the results that we see with xgboost.

README.md

@@ -7,6 +7,7 @@
 [![GitHub license](http://dmlc.github.io/img/apache2.svg)](./LICENSE)
 [![CRAN Status Badge](http://www.r-pkg.org/badges/version/xgboost)](http://cran.r-project.org/web/packages/xgboost)
 [![PyPI version](https://badge.fury.io/py/xgboost.svg)](https://pypi.python.org/pypi/xgboost/)
+[![Optuna](https://img.shields.io/badge/Optuna-integrated-blue)](https://optuna.org)
 
 [Community](https://xgboost.ai/community) |
 [Documentation](https://xgboost.readthedocs.org) |
@@ -17,11 +18,11 @@
 XGBoost is an optimized distributed gradient boosting library designed to be highly ***efficient***, ***flexible*** and ***portable***.
 It implements machine learning algorithms under the [Gradient Boosting](https://en.wikipedia.org/wiki/Gradient_boosting) framework.
 XGBoost provides a parallel tree boosting (also known as GBDT, GBM) that solve many data science problems in a fast and accurate way.
-The same code runs on major distributed environment (Hadoop, SGE, MPI) and can solve problems beyond billions of examples.
+The same code runs on major distributed environment (Kubernetes, Hadoop, SGE, MPI, Dask) and can solve problems beyond billions of examples.
 
 License
 -------
-© Contributors, 2016. Licensed under an [Apache-2](https://github.com/dmlc/xgboost/blob/master/LICENSE) license.
+© Contributors, 2019. Licensed under an [Apache-2](https://github.com/dmlc/xgboost/blob/master/LICENSE) license.
 
 Contribute to XGBoost
 ---------------------
@@ -38,7 +39,7 @@ Sponsors
 Become a sponsor and get a logo here. See details at [Sponsoring the XGBoost Project](https://xgboost.ai/sponsors). The funds are used to defray the cost of continuous integration and testing infrastructure (https://xgboost-ci.net).
 
 ## Open Source Collective sponsors
-[![Backers on Open Collective](https://opencollective.com/xgboost/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/xgboost/sponsors/badge.svg)](#sponsors) 
+[![Backers on Open Collective](https://opencollective.com/xgboost/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/xgboost/sponsors/badge.svg)](#sponsors)
 
 ### Sponsors
 [[Become a sponsor](https://opencollective.com/xgboost#sponsor)]

amalgamation/xgboost-all0.cc

@@ -1,5 +1,5 @@
 /*!
- * Copyright 2015 by Contributors.
+ * Copyright 2015-2019 by Contributors.
  * \brief XGBoost Amalgamation.
  *  This offers an alternative way to compile the entire library from this single file.
  *
@@ -25,25 +25,28 @@
 // gbms
 #include "../src/gbm/gbm.cc"
 #include "../src/gbm/gbtree.cc"
+#include "../src/gbm/gbtree_model.cc"
 #include "../src/gbm/gblinear.cc"
+#include "../src/gbm/gblinear_model.cc"
 
 // data
 #include "../src/data/data.cc"
 #include "../src/data/simple_csr_source.cc"
 #include "../src/data/simple_dmatrix.cc"
 #include "../src/data/sparse_page_raw_format.cc"
+#include "../src/data/ellpack_page.cc"
+#include "../src/data/ellpack_page_source.cc"
 
 // prediction
 #include "../src/predictor/predictor.cc"
 #include "../src/predictor/cpu_predictor.cc"
 
 #if DMLC_ENABLE_STD_THREAD
-#include "../src/data/sparse_page_source.cc"
 #include "../src/data/sparse_page_dmatrix.cc"
-#include "../src/data/sparse_page_writer.cc"
 #endif
 
 // tress
+#include "../src/tree/param.cc"
 #include "../src/tree/split_evaluator.cc"
 #include "../src/tree/tree_model.cc"
 #include "../src/tree/tree_updater.cc"
@@ -54,6 +57,7 @@
 #include "../src/tree/updater_sync.cc"
 #include "../src/tree/updater_histmaker.cc"
 #include "../src/tree/updater_skmaker.cc"
+#include "../src/tree/constraints.cc"
 
 // linear
 #include "../src/linear/linear_updater.cc"
@@ -64,8 +68,12 @@
 #include "../src/learner.cc"
 #include "../src/logging.cc"
 #include "../src/common/common.cc"
+#include "../src/common/timer.cc"
 #include "../src/common/host_device_vector.cc"
 #include "../src/common/hist_util.cc"
+#include "../src/common/json.cc"
+#include "../src/common/io.cc"
+#include "../src/common/version.cc"
 
 // c_api
 #include "../src/c_api/c_api.cc"

appveyor.yml

@@ -2,10 +2,6 @@ environment:
     R_ARCH: x64
     USE_RTOOLS: true
     matrix:
-        - target: msvc
-          ver: 2013
-          generator: "Visual Studio 12 2013 Win64"
-          configuration: Release
         - target: msvc
           ver: 2015
           generator: "Visual Studio 14 2015 Win64"
@@ -98,7 +94,7 @@ build_script:
         cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release" -DR_LIB=ON &&
         cmake --build . --target install --config Release
       )
-    - if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j
+    - if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j_2.12
 
 test_script:
     - cd %APPVEYOR_BUILD_FOLDER%
@@ -106,6 +102,7 @@ test_script:
     # mingw R package: run the R check (which includes unit tests), and also keep the built binary package
     - if /i "%target%" == "rmingw" (
         set _R_CHECK_CRAN_INCOMING_=FALSE&&
+        set _R_CHECK_FORCE_SUGGESTS_=FALSE&&
         R.exe CMD check xgboost*.tar.gz --no-manual --no-build-vignettes --as-cran --install-args=--build
       )
     # MSVC R package: run only the unit tests

cmake/Doc.cmake

@@ -6,7 +6,7 @@ function (run_doxygen)
   endif (NOT DOXYGEN_DOT_FOUND)
 
   configure_file(
-    ${PROJECT_SOURCE_DIR}/doc/Doxyfile.in
+    ${xgboost_SOURCE_DIR}/doc/Doxyfile.in
     ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
   add_custom_target( doc_doxygen ALL
     COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile

cmake/FindPrefetchIntrinsics.cmake

@@ -0,0 +1,22 @@
+function (find_prefetch_intrinsics)
+  include(CheckCXXSourceCompiles)
+  check_cxx_source_compiles("
+  #include <xmmintrin.h>
+  int main() {
+    char data = 0;
+    const char* address = &data;
+    _mm_prefetch(address, _MM_HINT_NTA);
+    return 0;
+  }
+  " XGBOOST_MM_PREFETCH_PRESENT)
+  check_cxx_source_compiles("
+  int main() {
+    char data = 0;
+    const char* address = &data;
+    __builtin_prefetch(address, 0, 0);
+    return 0;
+  }
+  " XGBOOST_BUILTIN_PREFETCH_PRESENT)
+  set(XGBOOST_MM_PREFETCH_PRESENT ${XGBOOST_MM_PREFETCH_PRESENT} PARENT_SCOPE)
+  set(XGBOOST_BUILTIN_PREFETCH_PRESENT ${XGBOOST_BUILTIN_PREFETCH_PRESENT} PARENT_SCOPE)
+endfunction (find_prefetch_intrinsics)

cmake/Python_version.in

@@ -0,0 +1 @@
+@xgboost_VERSION_MAJOR@.@xgboost_VERSION_MINOR@.@xgboost_VERSION_PATCH@

cmake/Sanitizer.cmake

@@ -4,24 +4,29 @@
 #  enable_sanitizers("address;leak")
 
 # Add flags
-macro(enable_sanitizer santizer)
-  if(${santizer} MATCHES "address")
+macro(enable_sanitizer sanitizer)
+  if(${sanitizer} MATCHES "address")
     find_package(ASan REQUIRED)
     set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address")
     link_libraries(${ASan_LIBRARY})
 
-  elseif(${santizer} MATCHES "thread")
+  elseif(${sanitizer} MATCHES "thread")
     find_package(TSan REQUIRED)
     set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread")
     link_libraries(${TSan_LIBRARY})
 
-  elseif(${santizer} MATCHES "leak")
+  elseif(${sanitizer} MATCHES "leak")
     find_package(LSan REQUIRED)
     set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak")
     link_libraries(${LSan_LIBRARY})
 
+  elseif(${sanitizer} MATCHES "undefined")
+    find_package(UBSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined")
+    link_libraries(${UBSan_LIBRARY})
+
   else()
-    message(FATAL_ERROR "Santizer ${santizer} not supported.")
+    message(FATAL_ERROR "Santizer ${sanitizer} not supported.")
   endif()
 endmacro()
 

cmake/Utils.cmake

@@ -58,9 +58,13 @@ function(set_output_directory target dir)
 		RUNTIME_OUTPUT_DIRECTORY ${dir}
 		RUNTIME_OUTPUT_DIRECTORY_DEBUG ${dir}
 		RUNTIME_OUTPUT_DIRECTORY_RELEASE ${dir}
+		RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
+		RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
 		LIBRARY_OUTPUT_DIRECTORY ${dir}
 		LIBRARY_OUTPUT_DIRECTORY_DEBUG ${dir}
 		LIBRARY_OUTPUT_DIRECTORY_RELEASE ${dir}
+		LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
+		LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
 	)
 endfunction(set_output_directory)
 
@@ -111,7 +115,7 @@ DESTINATION \"${build_dir}/bak\")")
 
   install(CODE "file(REMOVE_RECURSE \"${build_dir}/R-package\")")
   install(
-    DIRECTORY "${PROJECT_SOURCE_DIR}/R-package"
+    DIRECTORY "${xgboost_SOURCE_DIR}/R-package"
     DESTINATION "${build_dir}"
     REGEX "src/*" EXCLUDE
     REGEX "R-package/configure" EXCLUDE

cmake/Version.cmake

@@ -0,0 +1,9 @@
+function (write_version)
+  message(STATUS "xgboost VERSION: ${xgboost_VERSION}")
+  configure_file(
+    ${xgboost_SOURCE_DIR}/cmake/version_config.h.in
+    ${xgboost_SOURCE_DIR}/include/xgboost/version_config.h @ONLY)
+  configure_file(
+    ${xgboost_SOURCE_DIR}/cmake/Python_version.in
+    ${xgboost_SOURCE_DIR}/python-package/xgboost/VERSION @ONLY)
+endfunction (write_version)

cmake/modules/FindASan.cmake

@@ -1,7 +1,7 @@
 set(ASan_LIB_NAME ASan)
 
 find_library(ASan_LIBRARY
-  NAMES libasan.so libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0
+  NAMES libasan.so libasan.so.5 libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0
   PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
 
 include(FindPackageHandleStandardArgs)

cmake/modules/FindNVML.cmake

@@ -0,0 +1,23 @@
+if (NVML_LIBRARY)
+  unset(NVML_LIBRARY CACHE)
+endif(NVML_LIBRARY)
+
+set(NVML_LIB_NAME nvml)
+
+find_path(NVML_INCLUDE_DIR
+  NAMES nvml.h
+  PATHS ${CUDA_HOME}/include ${CUDA_INCLUDE} /usr/local/cuda/include)
+
+find_library(NVML_LIBRARY
+  NAMES nvidia-ml)
+
+message(STATUS "Using nvml library: ${NVML_LIBRARY}")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NVML DEFAULT_MSG
+                                  NVML_INCLUDE_DIR NVML_LIBRARY)
+
+mark_as_advanced(
+  NVML_INCLUDE_DIR
+  NVML_LIBRARY
+)

cmake/modules/FindUBSan.cmake

@@ -0,0 +1,13 @@
+set(UBSan_LIB_NAME UBSan)
+
+find_library(UBSan_LIBRARY
+  NAMES libubsan.so libubsan.so.5 libubsan.so.4 libubsan.so.3 libubsan.so.2 libubsan.so.1 libubsan.so.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(UBSan DEFAULT_MSG
+  UBSan_LIBRARY)
+
+mark_as_advanced(
+  UBSan_LIBRARY
+  UBSan_LIB_NAME)

cmake/version_config.h.in

@@ -0,0 +1,11 @@
+/*!
+ * Copyright 2019 XGBoost contributors
+ */
+#ifndef XGBOOST_VERSION_CONFIG_H_
+#define XGBOOST_VERSION_CONFIG_H_
+
+#define XGBOOST_VER_MAJOR @xgboost_VERSION_MAJOR@
+#define XGBOOST_VER_MINOR @xgboost_VERSION_MINOR@
+#define XGBOOST_VER_PATCH @xgboost_VERSION_PATCH@
+
+#endif  // XGBOOST_VERSION_CONFIG_H_

demo/c-api/CMakeLists.txt

@@ -0,0 +1,4 @@
+cmake_minimum_required(VERSION 3.12)
+find_package(xgboost REQUIRED)
+add_executable(api-demo c-api-demo.c)
+target_link_libraries(api-demo xgboost::xgboost)

demo/c-api/c-api-demo.c

@@ -36,13 +36,12 @@ int main(int argc, char** argv) {
   //   https://xgboost.readthedocs.io/en/latest/parameter.html
   safe_xgboost(XGBoosterSetParam(booster, "tree_method", use_gpu ? "gpu_hist" : "hist"));
   if (use_gpu) {
-    // set the number of GPUs and the first GPU to use;
+    // set the GPU to use;
     // this is not necessary, but provided here as an illustration
-    safe_xgboost(XGBoosterSetParam(booster, "n_gpus", "1"));
     safe_xgboost(XGBoosterSetParam(booster, "gpu_id", "0"));
   } else {
     // avoid evaluating objective and metric on a GPU
-    safe_xgboost(XGBoosterSetParam(booster, "n_gpus", "0"));
+    safe_xgboost(XGBoosterSetParam(booster, "gpu_id", "-1"));
   }
 
   safe_xgboost(XGBoosterSetParam(booster, "objective", "binary:logistic"));
@@ -66,7 +65,7 @@ int main(int argc, char** argv) {
   const float* out_result = NULL;
   int n_print = 10;
 
-  safe_xgboost(XGBoosterPredict(booster, dtest, 0, 0, &out_len, &out_result));
+  safe_xgboost(XGBoosterPredict(booster, dtest, 0, 0, 0, &out_len, &out_result));
   printf("y_pred: ");
   for (int i = 0; i < n_print; ++i) {
     printf("%1.4f ", out_result[i]);

demo/dask/README.md

@@ -0,0 +1,6 @@
+Dask
+====
+
+This directory contains some demonstrations for using `dask` with `XGBoost`.
+For an overview, see
+https://xgboost.readthedocs.io/en/latest/tutorials/dask.html .

demo/dask/cpu_training.py

@@ -0,0 +1,42 @@
+import xgboost as xgb
+from xgboost.dask import DaskDMatrix
+from dask.distributed import Client
+from dask.distributed import LocalCluster
+from dask import array as da
+
+
+def main(client):
+    # generate some random data for demonstration
+    m = 100000
+    n = 100
+    X = da.random.random(size=(m, n), chunks=100)
+    y = da.random.random(size=(m, ), chunks=100)
+
+    # DaskDMatrix acts like normal DMatrix, works as a proxy for local
+    # DMatrix scatter around workers.
+    dtrain = DaskDMatrix(client, X, y)
+
+    # Use train method from xgboost.dask instead of xgboost.  This
+    # distributed version of train returns a dictionary containing the
+    # resulting booster and evaluation history obtained from
+    # evaluation metrics.
+    output = xgb.dask.train(client,
+                            {'verbosity': 1,
+                             'nthread': 1,
+                             'tree_method': 'hist'},
+                            dtrain,
+                            num_boost_round=4, evals=[(dtrain, 'train')])
+    bst = output['booster']
+    history = output['history']
+
+    # you can pass output directly into `predict` too.
+    prediction = xgb.dask.predict(client, bst, dtrain)
+    print('Evaluation history:', history)
+    return prediction
+
+
+if __name__ == '__main__':
+    # or use other clusters for scaling
+    with LocalCluster(n_workers=7, threads_per_worker=1) as cluster:
+        with Client(cluster) as client:
+            main(client)

demo/dask/gpu_training.py

@@ -0,0 +1,46 @@
+from dask_cuda import LocalCUDACluster
+from dask.distributed import Client
+from dask import array as da
+import xgboost as xgb
+from xgboost.dask import DaskDMatrix
+
+
+def main(client):
+    # generate some random data for demonstration
+    m = 100000
+    n = 100
+    X = da.random.random(size=(m, n), chunks=100)
+    y = da.random.random(size=(m, ), chunks=100)
+
+    # DaskDMatrix acts like normal DMatrix, works as a proxy for local
+    # DMatrix scatter around workers.
+    dtrain = DaskDMatrix(client, X, y)
+
+    # Use train method from xgboost.dask instead of xgboost.  This
+    # distributed version of train returns a dictionary containing the
+    # resulting booster and evaluation history obtained from
+    # evaluation metrics.
+    output = xgb.dask.train(client,
+                            {'verbosity': 2,
+                             'nthread': 1,
+                             # Golden line for GPU training
+                             'tree_method': 'gpu_hist'},
+                            dtrain,
+                            num_boost_round=4, evals=[(dtrain, 'train')])
+    bst = output['booster']
+    history = output['history']
+
+    # you can pass output directly into `predict` too.
+    prediction = xgb.dask.predict(client, bst, dtrain)
+    prediction = prediction.compute()
+    print('Evaluation history:', history)
+    return prediction
+
+
+if __name__ == '__main__':
+    # `LocalCUDACluster` is used for assigning GPU to XGBoost processes.  Here
+    # `n_workers` represents the number of GPUs since we use one GPU per worker
+    # process.
+    with LocalCUDACluster(n_workers=2, threads_per_worker=1) as cluster:
+        with Client(cluster) as client:
+            main(client)

demo/dask/sklearn_cpu_training.py

@@ -0,0 +1,39 @@
+'''Dask interface demo:
+
+Use scikit-learn regressor interface with CPU histogram tree method.'''
+from dask.distributed import Client
+from dask.distributed import LocalCluster
+from dask import array as da
+import xgboost
+
+
+def main(client):
+    # generate some random data for demonstration
+    n = 100
+    m = 10000
+    partition_size = 100
+    X = da.random.random((m, n), partition_size)
+    y = da.random.random(m, partition_size)
+
+    regressor = xgboost.dask.DaskXGBRegressor(verbosity=1, n_estimators=2)
+    regressor.set_params(tree_method='hist')
+    # assigning client here is optional
+    regressor.client = client
+
+    regressor.fit(X, y, eval_set=[(X, y)])
+    prediction = regressor.predict(X)
+
+    bst = regressor.get_booster()
+    history = regressor.evals_result()
+
+    print('Evaluation history:', history)
+    # returned prediction is always a dask array.
+    assert isinstance(prediction, da.Array)
+    return bst                  # returning the trained model
+
+
+if __name__ == '__main__':
+    # or use other clusters for scaling
+    with LocalCluster(n_workers=4, threads_per_worker=1) as cluster:
+        with Client(cluster) as client:
+            main(client)

demo/dask/sklearn_gpu_training.py

@@ -0,0 +1,42 @@
+'''Dask interface demo:
+
+Use scikit-learn regressor interface with GPU histogram tree method.'''
+
+from dask.distributed import Client
+# It's recommended to use dask_cuda for GPU assignment
+from dask_cuda import LocalCUDACluster
+from dask import array as da
+import xgboost
+
+
+def main(client):
+    # generate some random data for demonstration
+    n = 100
+    m = 1000000
+    partition_size = 10000
+    X = da.random.random((m, n), partition_size)
+    y = da.random.random(m, partition_size)
+
+    regressor = xgboost.dask.DaskXGBRegressor(verbosity=1)
+    regressor.set_params(tree_method='gpu_hist')
+    # assigning client here is optional
+    regressor.client = client
+
+    regressor.fit(X, y, eval_set=[(X, y)])
+    prediction = regressor.predict(X)
+
+    bst = regressor.get_booster()
+    history = regressor.evals_result()
+
+    print('Evaluation history:', history)
+    # returned prediction is always a dask array.
+    assert isinstance(prediction, da.Array)
+    return bst                  # returning the trained model
+
+
+if __name__ == '__main__':
+    # With dask cuda, one can scale up XGBoost to arbitrary GPU clusters.
+    # `LocalCUDACluster` used here is only for demonstration purpose.
+    with LocalCUDACluster() as cluster:
+        with Client(cluster) as client:
+            main(client)

demo/gpu_acceleration/README.md

@@ -1,7 +1,5 @@
 # GPU Acceleration Demo
 
-This demo shows how to train a model on the [forest cover type](https://archive.ics.uci.edu/ml/datasets/covertype) dataset using GPU acceleration. The forest cover type dataset has 581,012 rows and 54 features, making it time consuming to process. We compare the run-time and accuracy of the GPU and CPU histogram algorithms.
+`cover_type.py` shows how to train a model on the [forest cover type](https://archive.ics.uci.edu/ml/datasets/covertype) dataset using GPU acceleration. The forest cover type dataset has 581,012 rows and 54 features, making it time consuming to process. We compare the run-time and accuracy of the GPU and CPU histogram algorithms.
 
-This demo requires the [GPU plug-in](https://xgboost.readthedocs.io/en/latest/gpu/index.html) to be built and installed.
-
-The dataset is automatically loaded via the sklearn script. 
+`memory.py` shows how to repeatedly train xgboost models while freeing memory between iterations.

demo/gpu_acceleration/memory.py

@@ -0,0 +1,51 @@
+import xgboost as xgb
+import numpy as np
+import time
+import pickle
+import GPUtil
+
+n = 10000
+m = 1000
+X = np.random.random((n, m))
+y = np.random.random(n)
+
+param = {'objective': 'binary:logistic',
+         'tree_method': 'gpu_hist'
+         }
+iterations = 5
+dtrain = xgb.DMatrix(X, label=y)
+
+# High memory usage
+# active bst objects with device memory persist across iterations
+boosters = []
+for i in range(iterations):
+    bst = xgb.train(param, dtrain)
+    boosters.append(bst)
+
+print("Example 1")
+GPUtil.showUtilization()
+del boosters
+
+# Better memory usage
+# The bst object can be destroyed by the python gc, freeing device memory
+# The gc may not immediately free the object, so more than one booster can be allocated at a time
+boosters = []
+for i in range(iterations):
+    bst = xgb.train(param, dtrain)
+    boosters.append(pickle.dumps(bst))
+
+print("Example 2")
+GPUtil.showUtilization()
+del boosters
+
+# Best memory usage
+# The gc explicitly frees the booster before starting the next iteration
+boosters = []
+for i in range(iterations):
+    bst = xgb.train(param, dtrain)
+    boosters.append(pickle.dumps(bst))
+    del bst
+
+print("Example 3")
+GPUtil.showUtilization()
+del boosters

demo/guide-python/README.md

@@ -2,6 +2,7 @@ XGBoost Python Feature Walkthrough
 ==================================
 * [Basic walkthrough of wrappers](basic_walkthrough.py)
 * [Customize loss function, and evaluation metric](custom_objective.py)
+* [Re-implement RMSLE as customized metric and objective](custom_rmsle.py)
 * [Boosting from existing prediction](boost_from_prediction.py)
 * [Predicting using first n trees](predict_first_ntree.py)
 * [Generalized Linear Model](generalized_linear_model.py)

demo/guide-python/boost_from_prediction.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-import numpy as np
 import xgboost as xgb
 
 dtrain = xgb.DMatrix('../data/agaricus.txt.train')
@@ -8,18 +7,19 @@ watchlist = [(dtest, 'eval'), (dtrain, 'train')]
 ###
 # advanced: start from a initial base prediction
 #
-print ('start running example to start from a initial prediction')
+print('start running example to start from a initial prediction')
 # specify parameters via map, definition are same as c++ version
-param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic'}
+param = {'max_depth': 2, 'eta': 1, 'silent': 1, 'objective': 'binary:logistic'}
 # train xgboost for 1 round
 bst = xgb.train(param, dtrain, 1, watchlist)
-# Note: we need the margin value instead of transformed prediction in set_base_margin
-# do predict with output_margin=True, will always give you margin values before logistic transformation
+# Note: we need the margin value instead of transformed prediction in
+# set_base_margin
+# do predict with output_margin=True, will always give you margin values
+# before logistic transformation
 ptrain = bst.predict(dtrain, output_margin=True)
 ptest = bst.predict(dtest, output_margin=True)
 dtrain.set_base_margin(ptrain)
 dtest.set_base_margin(ptest)
 
-
 print('this is result of running from initial prediction')
-bst = xgb.train(param, dtrain, 1, watchlist)
+bst = xgb.train(param, dtrain, 1, watchlist)

demo/guide-python/custom_objective.py

@@ -16,8 +16,9 @@ param = {'max_depth': 2, 'eta': 1, 'silent': 1}
 watchlist = [(dtest, 'eval'), (dtrain, 'train')]
 num_round = 2
 
-# user define objective function, given prediction, return gradient and second order gradient
-# this is log likelihood loss
+
+# user define objective function, given prediction, return gradient and second
+# order gradient this is log likelihood loss
 def logregobj(preds, dtrain):
     labels = dtrain.get_label()
     preds = 1.0 / (1.0 + np.exp(-preds))
@@ -25,18 +26,24 @@ def logregobj(preds, dtrain):
     hess = preds * (1.0 - preds)
     return grad, hess
 
+
 # user defined evaluation function, return a pair metric_name, result
-# NOTE: when you do customized loss function, the default prediction value is margin
-# this may make builtin evaluation metric not function properly
-# for example, we are doing logistic loss, the prediction is score before logistic transformation
-# the builtin evaluation error assumes input is after logistic transformation
-# Take this in mind when you use the customization, and maybe you need write customized evaluation function
+
+# NOTE: when you do customized loss function, the default prediction value is
+# margin. this may make builtin evaluation metric not function properly for
+# example, we are doing logistic loss, the prediction is score before logistic
+# transformation the builtin evaluation error assumes input is after logistic
+# transformation Take this in mind when you use the customization, and maybe
+# you need write customized evaluation function
 def evalerror(preds, dtrain):
     labels = dtrain.get_label()
-    # return a pair metric_name, result. The metric name must not contain a colon (:) or a space
-    # since preds are margin(before logistic transformation, cutoff at 0)
+    # return a pair metric_name, result. The metric name must not contain a
+    # colon (:) or a space since preds are margin(before logistic
+    # transformation, cutoff at 0)
     return 'my-error', float(sum(labels != (preds > 0.0))) / len(labels)
 
+
 # training with customized objective, we can also do step by step training
 # simply look at xgboost.py's implementation of train
-bst = xgb.train(param, dtrain, num_round, watchlist, obj=logregobj, feval=evalerror)
+bst = xgb.train(param, dtrain, num_round, watchlist, obj=logregobj,
+                feval=evalerror)

demo/guide-python/custom_rmsle.py

@@ -0,0 +1,179 @@
+'''Demo for defining customized metric and objective.  Notice that for
+simplicity reason weight is not used in following example. In this
+script, we implement the Squared Log Error (SLE) objective and RMSLE metric as customized
+functions, then compare it with native implementation in XGBoost.
+
+See doc/tutorials/custom_metric_obj.rst for a step by step
+walkthrough, with other details.
+
+The `SLE` objective reduces impact of outliers in training dataset,
+hence here we also compare its performance with standard squared
+error.
+
+'''
+import numpy as np
+import xgboost as xgb
+from typing import Tuple, Dict, List
+from time import time
+import matplotlib
+from matplotlib import pyplot as plt
+
+# shape of generated data.
+kRows = 4096
+kCols = 16
+
+kOutlier = 10000                # mean of generated outliers
+kNumberOfOutliers = 64
+
+kRatio = 0.7
+kSeed = 1994
+
+kBoostRound = 20
+
+np.random.seed(seed=kSeed)
+
+
+def generate_data() -> Tuple[xgb.DMatrix, xgb.DMatrix]:
+    '''Generate data containing outliers.'''
+    x = np.random.randn(kRows, kCols)
+    y = np.random.randn(kRows)
+    y += np.abs(np.min(y))
+
+    # Create outliers
+    for i in range(0, kNumberOfOutliers):
+        ind = np.random.randint(0, len(y)-1)
+        y[ind] += np.random.randint(0, kOutlier)
+
+    train_portion = int(kRows * kRatio)
+
+    # rmsle requires all label be greater than -1.
+    assert np.all(y > -1.0)
+
+    train_x: np.ndarray = x[: train_portion]
+    train_y: np.ndarray = y[: train_portion]
+    dtrain = xgb.DMatrix(train_x, label=train_y)
+
+    test_x = x[train_portion:]
+    test_y = y[train_portion:]
+    dtest = xgb.DMatrix(test_x, label=test_y)
+    return dtrain, dtest
+
+
+def native_rmse(dtrain: xgb.DMatrix,
+                dtest: xgb.DMatrix) -> Dict[str, Dict[str, List[float]]]:
+    '''Train using native implementation of Root Mean Squared Loss.'''
+    print('Squared Error')
+    squared_error = {
+        'objective': 'reg:squarederror',
+        'eval_metric': 'rmse',
+        'tree_method': 'hist',
+        'seed': kSeed
+    }
+    start = time()
+    results: Dict[str, Dict[str, List[float]]] = {}
+    xgb.train(squared_error,
+              dtrain=dtrain,
+              num_boost_round=kBoostRound,
+              evals=[(dtrain, 'dtrain'), (dtest, 'dtest')],
+              evals_result=results)
+    print('Finished Squared Error in:', time() - start, '\n')
+    return results
+
+
+def native_rmsle(dtrain: xgb.DMatrix,
+                 dtest: xgb.DMatrix) -> Dict[str, Dict[str, List[float]]]:
+    '''Train using native implementation of Squared Log Error.'''
+    print('Squared Log Error')
+    results: Dict[str, Dict[str, List[float]]] = {}
+    squared_log_error = {
+        'objective': 'reg:squaredlogerror',
+        'eval_metric': 'rmsle',
+        'tree_method': 'hist',
+        'seed': kSeed
+    }
+    start = time()
+    xgb.train(squared_log_error,
+              dtrain=dtrain,
+              num_boost_round=kBoostRound,
+              evals=[(dtrain, 'dtrain'), (dtest, 'dtest')],
+              evals_result=results)
+    print('Finished Squared Log Error in:', time() - start)
+    return results
+
+
+def py_rmsle(dtrain: xgb.DMatrix, dtest: xgb.DMatrix) -> Dict:
+    '''Train using Python implementation of Squared Log Error.'''
+    def gradient(predt: np.ndarray, dtrain: xgb.DMatrix) -> np.ndarray:
+        '''Compute the gradient squared log error.'''
+        y = dtrain.get_label()
+        return (np.log1p(predt) - np.log1p(y)) / (predt + 1)
+
+    def hessian(predt: np.ndarray, dtrain: xgb.DMatrix) -> np.ndarray:
+        '''Compute the hessian for squared log error.'''
+        y = dtrain.get_label()
+        return ((-np.log1p(predt) + np.log1p(y) + 1) /
+                np.power(predt + 1, 2))
+
+    def squared_log(predt: np.ndarray,
+                    dtrain: xgb.DMatrix) -> Tuple[np.ndarray, np.ndarray]:
+        '''Squared Log Error objective. A simplified version for RMSLE used as
+        objective function.
+
+        :math:`\frac{1}{2}[log(pred + 1) - log(label + 1)]^2`
+
+        '''
+        predt[predt < -1] = -1 + 1e-6
+        grad = gradient(predt, dtrain)
+        hess = hessian(predt, dtrain)
+        return grad, hess
+
+    def rmsle(predt: np.ndarray, dtrain: xgb.DMatrix) -> Tuple[str, float]:
+        ''' Root mean squared log error metric.
+
+        :math:`\sqrt{\frac{1}{N}[log(pred + 1) - log(label + 1)]^2}`
+        '''
+        y = dtrain.get_label()
+        predt[predt < -1] = -1 + 1e-6
+        elements = np.power(np.log1p(y) - np.log1p(predt), 2)
+        return 'PyRMSLE', float(np.sqrt(np.sum(elements) / len(y)))
+
+    results: Dict[str, Dict[str, List[float]]] = {}
+    xgb.train({'tree_method': 'hist', 'seed': kSeed,
+               'disable_default_eval_metric': 1},
+              dtrain=dtrain,
+              num_boost_round=kBoostRound,
+              obj=squared_log,
+              feval=rmsle,
+              evals=[(dtrain, 'dtrain'), (dtest, 'dtest')],
+              evals_result=results)
+
+    return results
+
+
+if __name__ == '__main__':
+    dtrain, dtest = generate_data()
+    rmse_evals = native_rmse(dtrain, dtest)
+    rmsle_evals = native_rmsle(dtrain, dtest)
+    py_rmsle_evals = py_rmsle(dtrain, dtest)
+
+    fig, axs = plt.subplots(3, 1)
+    ax0: matplotlib.axes.Axes = axs[0]
+    ax1: matplotlib.axes.Axes = axs[1]
+    ax2: matplotlib.axes.Axes = axs[2]
+
+    x = np.arange(0, kBoostRound, 1)
+
+    ax0.plot(x, rmse_evals['dtrain']['rmse'], label='train-RMSE')
+    ax0.plot(x, rmse_evals['dtest']['rmse'], label='test-RMSE')
+    ax0.legend()
+
+    ax1.plot(x, rmsle_evals['dtrain']['rmsle'], label='train-native-RMSLE')
+    ax1.plot(x, rmsle_evals['dtest']['rmsle'], label='test-native-RMSLE')
+    ax1.legend()
+
+    ax2.plot(x, py_rmsle_evals['dtrain']['PyRMSLE'], label='train-PyRMSLE')
+    ax2.plot(x, py_rmsle_evals['dtest']['PyRMSLE'], label='test-PyRMSLE')
+    ax2.legend()
+
+    plt.show()
+    plt.close()

demo/json-model/README.md

@@ -0,0 +1,3 @@
+We introduced initial support for saving XGBoost model in JSON format in 1.0.0.  Note that
+it's still experimental and under development, output schema is subject to change due to
+bug fixes or further refactoring.  For an overview, see https://xgboost.readthedocs.io/en/latest/tutorials/saving_model.html .

demo/json-model/json_parser.py

@@ -0,0 +1,180 @@
+'''Demonstration for parsing JSON tree model file generated by XGBoost.  The
+support is experimental, output schema is subject to change in the future.
+'''
+import json
+import argparse
+
+
+class Tree:
+    '''A tree built by XGBoost.'''
+    # Index into node array
+    _left = 0
+    _right = 1
+    _parent = 2
+    _ind = 3
+    _cond = 4
+    _default_left = 5
+    # Index into stat array
+    _loss_chg = 0
+    _sum_hess = 1
+    _base_weight = 2
+    _child_cnt = 3
+
+    def __init__(self, tree_id: int, nodes, stats):
+        self.tree_id = tree_id
+        self.nodes = nodes
+        self.stats = stats
+
+    def loss_change(self, node_id: int):
+        '''Loss gain of a node.'''
+        return self.stats[node_id][self._loss_chg]
+
+    def sum_hessian(self, node_id: int):
+        '''Sum Hessian of a node.'''
+        return self.stats[node_id][self._sum_hess]
+
+    def base_weight(self, node_id: int):
+        '''Base weight of a node.'''
+        return self.stats[node_id][self._base_weight]
+
+    def num_children(self, node_id: int):
+        '''Number of children of a node.'''
+        return self.stats[node_id][self._child_cnt]
+
+    def split_index(self, node_id: int):
+        '''Split feature index of node.'''
+        return self.nodes[node_id][self._ind]
+
+    def split_condition(self, node_id: int):
+        '''Split value of a node.'''
+        return self.nodes[node_id][self._cond]
+
+    def parent(self, node_id: int):
+        '''Parent ID of a node.'''
+        return self.nodes[node_id][self._parent]
+
+    def left_child(self, node_id: int):
+        '''Left child ID of a node.'''
+        return self.nodes[node_id][self._left]
+
+    def right_child(self, node_id: int):
+        '''Right child ID of a node.'''
+        return self.nodes[node_id][self._right]
+
+    def is_leaf(self, node_id: int):
+        '''Whether a node is leaf.'''
+        return self.nodes[node_id][self._left] == -1
+
+    def is_deleted(self, node_id: int):
+        '''Whether a node is deleted.'''
+        # std::numeric_limits<uint32_t>::max()
+        return self.nodes[node_id][self._ind] == 4294967295
+
+    def __str__(self):
+        stacks = [0]
+        nodes = []
+        while stacks:
+            node = {}
+            nid = stacks.pop()
+
+            node['node id'] = nid
+            node['gain'] = self.loss_change(nid)
+            node['cover'] = self.sum_hessian(nid)
+            nodes.append(node)
+
+            if not self.is_leaf(nid) and not self.is_deleted(nid):
+                left = self.left_child(nid)
+                right = self.right_child(nid)
+                stacks.append(left)
+                stacks.append(right)
+
+        string = '\n'.join(map(lambda x: '  ' + str(x), nodes))
+        return string
+
+
+class Model:
+    '''Gradient boosted tree model.'''
+    def __init__(self, m: dict):
+        '''Construct the Model from JSON object.
+
+         parameters
+         ----------
+          m: A dictionary loaded by json
+        '''
+        # Basic property of a model
+        self.learner_model_shape = model['learner']['learner_model_param']
+        self.num_output_group = int(self.learner_model_shape['num_class'])
+        self.num_feature = int(self.learner_model_shape['num_feature'])
+        self.base_score = float(self.learner_model_shape['base_score'])
+        # A field encoding which output group a tree belongs
+        self.tree_info = model['learner']['gradient_booster']['model'][
+            'tree_info']
+
+        model_shape = model['learner']['gradient_booster']['model'][
+            'gbtree_model_param']
+
+        # JSON representation of trees
+        j_trees = model['learner']['gradient_booster']['model']['trees']
+
+        # Load the trees
+        self.num_trees = int(model_shape['num_trees'])
+        self.leaf_size = int(model_shape['size_leaf_vector'])
+        # Right now XGBoost doesn't support vector leaf yet
+        assert self.leaf_size == 0, str(self.leaf_size)
+
+        trees = []
+        for i in range(self.num_trees):
+            tree = j_trees[i]
+            tree_id = int(tree['id'])
+            assert tree_id == i, (tree_id, i)
+            # properties
+            left_children = tree['left_children']
+            right_children = tree['right_children']
+            parents = tree['parents']
+            split_conditions = tree['split_conditions']
+            split_indices = tree['split_indices']
+            default_left = tree['default_left']
+            # stats
+            base_weights = tree['base_weights']
+            loss_changes = tree['loss_changes']
+            sum_hessian = tree['sum_hessian']
+            leaf_child_counts = tree['leaf_child_counts']
+
+            stats = []
+            nodes = []
+            # We resemble the structure used inside XGBoost, which is similar
+            # to adjacency list.
+            for node_id in range(len(left_children)):
+                nodes.append([
+                    left_children[node_id], right_children[node_id],
+                    parents[node_id], split_indices[node_id],
+                    split_conditions[node_id], default_left[node_id]
+                ])
+                stats.append([
+                    loss_changes[node_id], sum_hessian[node_id],
+                    base_weights[node_id], leaf_child_counts[node_id]
+                ])
+
+            tree = Tree(tree_id, nodes, stats)
+            trees.append(tree)
+
+        self.trees = trees
+
+    def print_model(self):
+        for i, tree in enumerate(self.trees):
+            print('tree_id:', i)
+            print(tree)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Demonstration for loading and printing XGBoost model.')
+    parser.add_argument('--model',
+                        type=str,
+                        required=True,
+                        help='Path to JSON model file.')
+    args = parser.parse_args()
+    with open(args.model, 'r') as fd:
+        model = json.load(fd)
+    model = Model(model)
+    model.print_model()

demo/rank/README.md

@@ -1,6 +1,6 @@
 Learning to rank
 ====
-XGBoost supports accomplishing ranking tasks. In ranking scenario, data are often grouped and we need the [group information file](../../doc/tutorials/input_format.rst#group-input-format) to specify ranking tasks. The model used in XGBoost for ranking is the LambdaRank, this function is not yet completed. Currently, we provide pairwise rank.
+XGBoost supports accomplishing ranking tasks. In ranking scenario, data are often grouped and we need the [group information file](../../doc/tutorials/input_format.rst#group-input-format) to specify ranking tasks. The model used in XGBoost for ranking is the LambdaRank. See [parameters](../../doc/parameter.rst) for supported metrics.
 
 ### Parameters
 The configuration setting is similar to the regression and binary classification setting, except user need to specify the objectives:
@@ -28,7 +28,7 @@ Run the example:
 ```
 
 ### Python
-There are two ways of doing ranking in python.  
+There are two ways of doing ranking in python.
 
 Run the example using `xgboost.train`:
 ```

demo/rank/rank.py

@@ -34,8 +34,8 @@ test_dmatrix = DMatrix(x_test)
 train_dmatrix.set_group(group_train)
 valid_dmatrix.set_group(group_valid)
 
-params = {'objective': 'rank:pairwise', 'eta': 0.1, 'gamma': 1.0,
-               'min_child_weight': 0.1, 'max_depth': 6}
+params = {'objective': 'rank:ndcg', 'eta': 0.1, 'gamma': 1.0,
+          'min_child_weight': 0.1, 'max_depth': 6}
 xgb_model = xgb.train(params, train_dmatrix, num_boost_round=4,
-                           evals=[(valid_dmatrix, 'validation')])
+                      evals=[(valid_dmatrix, 'validation')])
 pred = xgb_model.predict(test_dmatrix)

demo/rank/rank_sklearn.py

@@ -2,7 +2,6 @@
 import xgboost as xgb
 from sklearn.datasets import load_svmlight_file
 
-
 #  This script demonstrate how to do ranking with XGBRanker
 x_train, y_train = load_svmlight_file("mq2008.train")
 x_valid, y_valid = load_svmlight_file("mq2008.vali")
@@ -26,10 +25,10 @@ with open("mq2008.test.group", "r") as f:
     for line in data:
         group_test.append(int(line.split("\n")[0]))
 
-params = {'objective': 'rank:pairwise', 'learning_rate': 0.1,
+params = {'objective': 'rank:ndcg', 'learning_rate': 0.1,
           'gamma': 1.0, 'min_child_weight': 0.1,
           'max_depth': 6, 'n_estimators': 4}
 model = xgb.sklearn.XGBRanker(**params)
-model.fit(x_train, y_train, group_train,
+model.fit(x_train, y_train, group_train, verbose=True,
           eval_set=[(x_valid, y_valid)], eval_group=[group_valid])
 pred = model.predict(x_test)

doc/R-package/xgboostPresentation.md

@@ -44,7 +44,7 @@ drat:::addRepo("dmlc")
 install.packages("xgboost", repos="http://dmlc.ml/drat/", type = "source")
 ```
 
-> *Windows* user will need to install [Rtools](http://cran.r-project.org/bin/windows/Rtools/) first.
+> *Windows* users will need to install [Rtools](http://cran.r-project.org/bin/windows/Rtools/) first.
 
 ### CRAN version
 
@@ -97,7 +97,7 @@ train <- agaricus.train
 test <- agaricus.test
 ```
 
-> In the real world, it would be up to you to make this division between `train` and `test` data. The way to do it is out of the purpose of this article, however `caret` package may [help](http://topepo.github.io/caret/splitting.html).
+> In the real world, it would be up to you to make this division between `train` and `test` data. The way to do it is out of scope for this article, however `caret` package may [help](http://topepo.github.io/caret/data-splitting.html).
 
 Each variable is a `list` containing two things, `label` and `data`:
 
@@ -141,7 +141,7 @@ dim(test$data)
 ## [1] 1611  126
 ```
 
-This dataset is very small to not make the **R** package too heavy, however **XGBoost** is built to manage huge dataset very efficiently.
+This dataset is very small to not make the **R** package too heavy, however **XGBoost** is built to manage huge datasets very efficiently.
 
 As seen below, the `data` are stored in a `dgCMatrix` which is a *sparse* matrix and `label` vector is a `numeric` vector (`{0,1}`):
 
@@ -171,7 +171,7 @@ This step is the most critical part of the process for the quality of our model.
 
 We are using the `train` data. As explained above, both `data` and `label` are stored in a `list`.
 
-In a *sparse* matrix, cells containing `0` are not stored in memory. Therefore, in a dataset mainly made of `0`, memory size is reduced. It is very usual to have such dataset.
+In a *sparse* matrix, cells containing `0` are not stored in memory. Therefore, in a dataset mainly made of `0`, memory size is reduced. It is very common to have such a dataset.
 
 We will train decision tree model using the following parameters:
 
@@ -190,7 +190,7 @@ bstSparse <- xgboost(data = train$data, label = train$label, max.depth = 2, eta
 ## [1]	train-error:0.022263
 ```
 
-> More complex the relationship between your features and your `label` is, more passes you need.
+> The more complex the relationship between your features and your `label` is, the more passes you need.
 
 #### Parameter variations
 
@@ -210,7 +210,7 @@ bstDense <- xgboost(data = as.matrix(train$data), label = train$label, max.depth
 
 ##### xgb.DMatrix
 
-**XGBoost** offers a way to group them in a `xgb.DMatrix`. You can even add other meta data in it. It will be useful for the most advanced features we will discover later.
+**XGBoost** offers a way to group them in a `xgb.DMatrix`. You can even add other meta data in it. This will be useful for the most advanced features we will discover later.
 
 
 ```r
@@ -225,9 +225,9 @@ bstDMatrix <- xgboost(data = dtrain, max.depth = 2, eta = 1, nthread = 2, nround
 
 ##### Verbose option
 
-**XGBoost** has several features to help you to view how the learning progress internally. The purpose is to help you to set the best parameters, which is the key of your model quality.
+**XGBoost** has several features to help you view the learning progress internally. The purpose is to help you to set the best parameters, which is the key of your model quality.
 
-One of the simplest way to see the training progress is to set the `verbose` option (see below for more advanced technics).
+One of the simplest way to see the training progress is to set the `verbose` option (see below for more advanced techniques).
 
 
 ```r
@@ -360,11 +360,11 @@ dtest <- xgb.DMatrix(data = test$data, label=test$label)
 
 Both `xgboost` (simple) and `xgb.train` (advanced) functions train models.
 
-One of the special feature of `xgb.train` is the capacity to follow the progress of the learning after each round. Because of the way boosting works, there is a time when having too many rounds lead to an overfitting. You can see this feature as a cousin of cross-validation method. The following techniques will help you to avoid overfitting or optimizing the learning time in stopping it as soon as possible.
+One of the special features of `xgb.train` is the capacity to follow the progress of the learning after each round. Because of the way boosting works, there is a time when having too many rounds lead to overfitting. You can see this feature as a cousin of a cross-validation method. The following techniques will help you to avoid overfitting or optimizing the learning time in stopping it as soon as possible.
 
-One way to measure progress in learning of a model is to provide to **XGBoost** a second dataset already classified. Therefore it can learn on the first dataset and test its model on the second one. Some metrics are measured after each round during the learning.
+One way to measure progress in the learning of a model is to provide to **XGBoost** a second dataset already classified. Therefore it can learn on the first dataset and test its model on the second one. Some metrics are measured after each round during the learning.
 
-> in some way it is similar to what we have done above with the average error. The main difference is that below it was after building the model, and now it is during the construction that we measure errors.
+> in some way it is similar to what we have done above with the average error. The main difference is that above it was after building the model, and now it is during the construction that we measure errors.
 
 For the purpose of this example, we use `watchlist` parameter. It is a list of `xgb.DMatrix`, each of them tagged with a name.
 
@@ -380,11 +380,11 @@ bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nthread = 2, nrounds=2, watchl
 ## [1]	train-error:0.022263	test-error:0.021726
 ```
 
-**XGBoost** has computed at each round the same average error metric than seen above (we set `nrounds` to 2, that is why we have two lines). Obviously, the `train-error` number is related to the training dataset (the one the algorithm learns from) and the `test-error` number to the test dataset.
+**XGBoost** has computed at each round the same average error metric seen above (we set `nrounds` to 2, that is why we have two lines). Obviously, the `train-error` number is related to the training dataset (the one the algorithm learns from) and the `test-error` number to the test dataset.
 
 Both training and test error related metrics are very similar, and in some way, it makes sense: what we have learned from the training dataset matches the observations from the test dataset.
 
-If with your own dataset you have not such results, you should think about how you divided your dataset in training and test. May be there is something to fix. Again, `caret` package may [help](http://topepo.github.io/caret/splitting.html).
+If with your own dataset you do not have such results, you should think about how you divided your dataset in training and test. May be there is something to fix. Again, `caret` package may [help](http://topepo.github.io/caret/data-splitting.html).
 
 For a better understanding of the learning progression, you may want to have some specific metric or even use multiple evaluation metrics.
 
@@ -403,7 +403,7 @@ bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nthread = 2, nrounds=2, watchl
 ### Linear boosting
 
 
-Until now, all the learnings we have performed were based on boosting trees. **XGBoost** implements a second algorithm, based on linear boosting. The only difference with previous command is `booster = "gblinear"` parameter (and removing `eta` parameter).
+Until now, all the learnings we have performed were based on boosting trees. **XGBoost** implements a second algorithm, based on linear boosting. The only difference with the previous command is `booster = "gblinear"` parameter (and removing `eta` parameter).
 
 
 ```r
@@ -415,9 +415,9 @@ bst <- xgb.train(data=dtrain, booster = "gblinear", max.depth=2, nthread = 2, nr
 ## [1]	train-error:0.004146	train-logloss:0.069885	test-error:0.003724	test-logloss:0.068081
 ```
 
-In this specific case, *linear boosting* gets slightly better performance metrics than decision trees based algorithm.
+In this specific case, *linear boosting* gets slightly better performance metrics than a decision tree based algorithm.
 
-In simple cases, it will happen because there is nothing better than a linear algorithm to catch a linear link. However, decision trees are much better to catch a non linear link between predictors and outcome. Because there is no silver bullet, we advise you to check both algorithms with your own datasets to have an idea of what to use.
+In simple cases, this will happen because there is nothing better than a linear algorithm to catch a linear link. However, decision trees are much better to catch a non linear link between predictors and outcome. Because there is no silver bullet, we advise you to check both algorithms with your own datasets to have an idea of what to use.
 
 ### Manipulating xgb.DMatrix
 
@@ -457,7 +457,7 @@ bst <- xgb.train(data=dtrain2, max.depth=2, eta=1, nthread = 2, nrounds=2, watch
 
 #### Information extraction
 
-Information can be extracted from `xgb.DMatrix` using `getinfo` function. Hereafter we will extract `label` data.
+Information can be extracted from an `xgb.DMatrix` using `getinfo` function. Hereafter we will extract `label` data.
 
 
 ```r
@@ -489,7 +489,7 @@ You can dump the tree you learned using `xgb.dump` into a text file.
 
 
 ```r
-xgb.dump(bst, with.stats = T)
+xgb.dump(bst, with_stats = T)
 ```
 
 ```
@@ -522,7 +522,7 @@ xgb.plot.tree(model = bst)
 
 Maybe your dataset is big, and it takes time to train a model on it? May be you are not a big fan of losing time in redoing the same task again and again? In these very rare cases, you will want to save your model and load it when required.
 
-Hopefully for you, **XGBoost** implements such functions.
+Helpfully for you, **XGBoost** implements such functions.
 
 
 ```r

doc/build.rst

@@ -13,8 +13,9 @@ Installation Guide
     #   * xgboost-{version}-py2.py3-none-win_amd64.whl
     pip3 install xgboost
 
-  * The binary wheel will support GPU algorithms (`gpu_exact`, `gpu_hist`) on machines with NVIDIA GPUs. Please note that **training with multiple GPUs is only supported for Linux platform**. See :doc:`gpu/index`.
+  * The binary wheel will support GPU algorithms (`gpu_hist`) on machines with NVIDIA GPUs. Please note that **training with multiple GPUs is only supported for Linux platform**. See :doc:`gpu/index`.
   * Currently, we provide binary wheels for 64-bit Linux and Windows.
+  * Nightly builds are available. You can now run *pip install https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/xgboost-[version]+[commit hash]-py2.py3-none-manylinux1_x86_64.whl* to install the nightly build with the given commit hash. See `this page <https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/list.html>`_ to see the list of all nightly builds.
 
 ****************************
 Building XGBoost from source
@@ -57,6 +58,9 @@ to ask questions at `the user forum <https://discuss.xgboost.ai>`_.
 * `Python Package Installation`_
 * `R Package Installation`_
 * `Trouble Shooting`_
+* `Building the documentation`_
+
+.. _build_shared_lib:
 
 ***************************
 Building the Shared Library
@@ -69,8 +73,10 @@ Our goal is to build the shared library:
 
 The minimal building requirement is
 
-- A recent C++ compiler supporting C++11 (g++-4.8 or higher)
-- CMake 3.2 or higher
+- A recent C++ compiler supporting C++11 (g++-5.0 or higher)
+- CMake 3.3 or higher (3.12 for building with CUDA)
+
+For a list of CMake options, see ``#-- Options`` in CMakeLists.txt on top of source tree.
 
 Building on Ubuntu/Debian
 =========================
@@ -92,11 +98,11 @@ Building on OSX
 Install with pip: simple method
 --------------------------------
 
-First, obtain ``gcc-8`` with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training). The default Apple Clang compiler does not support OpenMP, so using the default compiler would have disabled multi-threading.
+First, obtain the OpenMP library (``libomp``) with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training):
 
 .. code-block:: bash
 
-  brew install gcc@8
+  brew install libomp
 
 Then install XGBoost with ``pip``:
 
@@ -109,11 +115,12 @@ You might need to run the command with ``--user`` flag if you run into permissio
 Build from the source code - advanced method
 --------------------------------------------
 
-Obtain ``gcc-8`` from Homebrew:
+Obtain ``libomp`` from Homebrew:
 
 .. code-block:: bash
 
-  brew install gcc@8
+  brew install libomp
+
 
 Now clone the repository:
 
@@ -121,13 +128,13 @@ Now clone the repository:
 
   git clone --recursive https://github.com/dmlc/xgboost
 
-Create the ``build/`` directory and invoke CMake. Make sure to add ``CC=gcc-8 CXX=g++-8`` so that Homebrew GCC is selected. After invoking CMake, you can build XGBoost with ``make``:
+Create the ``build/`` directory and invoke CMake. After invoking CMake, you can build XGBoost with ``make``:
 
 .. code-block:: bash
 
   mkdir build
   cd build
-  CC=gcc-8 CXX=g++-8 cmake ..
+  cmake ..
   make -j4
 
 You may now continue to `Python Package Installation`_.
@@ -185,9 +192,7 @@ Building with GPU support
 =========================
 XGBoost can be built with GPU support for both Linux and Windows using CMake. GPU support works with the Python package as well as the CLI version. See `Installing R package with GPU support`_ for special instructions for R.
 
-An up-to-date version of the CUDA toolkit is required.  Please note that we
-skipped the support for compiling XGBoost with NVCC 10.1 due a small bug in its
-spliter, see `#4264 <https://github.com/dmlc/xgboost/issues/4264>`_.
+An up-to-date version of the CUDA toolkit is required.
 
 From the command line on Linux starting from the XGBoost directory:
 
@@ -198,9 +203,9 @@ From the command line on Linux starting from the XGBoost directory:
   cmake .. -DUSE_CUDA=ON
   make -j4
 
-.. note:: Enabling multi-GPU training
+.. note:: Enabling distributed GPU training
 
-  By default, multi-GPU training is disabled and only a single GPU will be used. To enable multi-GPU training, set the option ``USE_NCCL=ON``. Multi-GPU training depends on NCCL2, available at `this link <https://developer.nvidia.com/nccl>`_. Since NCCL2 is only available for Linux machines, **multi-GPU training is available only for Linux**.
+  By default, distributed GPU training is disabled and only a single GPU will be used. To enable distributed GPU training, set the option ``USE_NCCL=ON``. Distributed GPU training depends on NCCL2, available at `this link <https://developer.nvidia.com/nccl>`_. Since NCCL2 is only available for Linux machines, **distributed GPU training is available only for Linux**.
 
   .. code-block:: bash
 
@@ -225,7 +230,7 @@ On Windows, run CMake as follows:
 
   .. code-block:: bash
 
-    make .. -G"Visual Studio 15 2017 Win64" -T v140,cuda=8.0 -DUSE_CUDA=ON
+    cmake .. -G"Visual Studio 15 2017 Win64" -T v140,cuda=8.0 -DUSE_CUDA=ON
 
 To speed up compilation, the compute version specific to your GPU could be passed to cmake as, e.g., ``-DGPU_COMPUTE_VER=50``.
 The above cmake configuration run will create an ``xgboost.sln`` solution file in the build directory. Build this solution in release mode as a x64 build, either from Visual studio or from command line:
@@ -311,13 +316,21 @@ R Package Installation
 Installing pre-packaged version
 -------------------------------
 
-You can install xgboost from CRAN just like any other R package:
+You can install XGBoost from CRAN just like any other R package:
 
 .. code-block:: R
 
-  install.packages("xgboost")
+   install.packages("xgboost")
 
-For OSX users, single-threaded version will be installed. So only one thread will be used for training. To enable use of multiple threads (and utilize capacity of multi-core CPUs), see the section :ref:`osx_multithread` to install XGBoost from source.
+.. note:: Using all CPU cores (threads) on Mac OSX
+
+   If you are using Mac OSX, you should first install OpenMP library (``libomp``) by running
+
+   .. code-block:: bash
+
+      brew install libomp
+
+   and then run ``install.packages("xgboost")``. Without OpenMP, XGBoost will only use a single CPU core, leading to suboptimal training speed.
 
 Installing the development version
 ----------------------------------
@@ -334,48 +347,14 @@ Thus, one has to run git to check out the code first:
   cd xgboost
   git submodule init
   git submodule update
-  cd R-package
-  R CMD INSTALL .
-
-If the last line fails because of the error ``R: command not found``, it means that R was not set up to run from command line.
-In this case, just start R as you would normally do and run the following:
-
-.. code-block:: R
-
-  setwd('wherever/you/cloned/it/xgboost/R-package/')
-  install.packages('.', repos = NULL, type="source")
-
-The package could also be built and installed with CMake (and Visual C++ 2015 on Windows) using instructions from :ref:`r_gpu_support`, but without GPU support (omit the ``-DUSE_CUDA=ON`` cmake parameter).
-
-If all fails, try `Building the shared library`_ to see whether a problem is specific to R package or not.
-
-.. _osx_multithread:
-
-Installing R package on Mac OSX with multi-threading
-----------------------------------------------------
-
-First, obtain ``gcc-8`` with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training). The default Apple Clang compiler does not support OpenMP, so using the default compiler would have disabled multi-threading.
-
-.. code-block:: bash
-
-  brew install gcc@8
-
-Now, clone the repository:
-
-.. code-block:: bash
-
-  git clone --recursive https://github.com/dmlc/xgboost
-
-Create the ``build/`` directory and invoke CMake with option ``R_LIB=ON``. Make sure to add ``CC=gcc-8 CXX=g++-8`` so that Homebrew GCC is selected. After invoking CMake, you can install the R package by running ``make`` and ``make install``:
-
-.. code-block:: bash
-
   mkdir build
   cd build
-  CC=gcc-7 CXX=g++-7 cmake .. -DR_LIB=ON
+  cmake .. -DR_LIB=ON
   make -j4
   make install
 
+If all fails, try `Building the shared library`_ to see whether a problem is specific to R package or not.
+
 .. _r_gpu_support:
 
 Installing R package with GPU support
@@ -450,3 +429,23 @@ Trouble Shooting
    .. code-block:: bash
 
      git clone https://github.com/dmlc/xgboost --recursive
+
+
+Building the Documentation
+==========================
+XGBoost uses `Sphinx <https://www.sphinx-doc.org/en/stable/>`_ for documentation.  To build it locally, you need a installed XGBoost with all its dependencies along with:
+
+* System dependencies
+
+  - git
+  - graphviz
+
+* Python dependencies
+
+  - sphinx
+  - breathe
+  - guzzle_sphinx_theme
+  - recommonmark
+  - mock
+
+Under ``xgboost/doc`` directory, run ``make <format>`` with ``<format>`` replaced by the format you want.  For a list of supported formats, run ``make help`` under the same directory.

doc/c++.rst

@@ -0,0 +1,12 @@
+###############
+XGBoost C++ API
+###############
+
+Starting from 1.0 release, CMake will generate installation rules to export all C++ headers. But
+the c++ interface is much closer to the internal of XGBoost than other language bindings.
+As a result it's changing quite often and we don't maintain its stability.  Along with the
+plugin system (see ``plugin/example`` in XGBoost's source tree), users can utilize some
+existing c++ headers for gaining more access to the internal of XGBoost.
+
+* `C++ interface documentation (latest master branch) <https://xgboost.readthedocs.io/en/latest/dev/files.html>`_
+* `C++ interface documentation (last stable release) <https://xgboost.readthedocs.io/en/stable/dev/files.html>`_

doc/c.rst

@@ -0,0 +1,12 @@
+#################
+XGBoost C Package
+#################
+
+XGBoost implements a set of C API designed for various bindings, we maintain its
+stability and the CMake/make build interface.  See ``demo/c-api/README.md`` for an
+overview and related examples.  Also one can generate doxygen document by providing
+``-DBUILD_C_DOC=ON`` as parameter to ``CMake`` during build, or simply look at function
+comments in ``include/xgboost/c_api.h``.
+
+* `C API documentation (latest master branch) <https://xgboost.readthedocs.io/en/latest/dev/c__api_8h.html>`_
+* `C API documentation (last stable release) <https://xgboost.readthedocs.io/en/stable/dev/c__api_8h.html>`_

doc/conf.py

@@ -58,7 +58,7 @@ for mod_name in MOCK_MODULES:
 # General information about the project.
 project = u'xgboost'
 author = u'%s developers' % project
-copyright = u'2016, %s' % author
+copyright = u'2019, %s' % author
 github_doc_root = 'https://github.com/dmlc/xgboost/tree/master/doc/'
 
 os.environ['XGBOOST_BUILD_DOC'] = '1'

doc/contrib/coding_guide.rst

@@ -0,0 +1,136 @@
+################
+Coding Guideline
+################
+
+**Contents**
+
+.. contents::
+  :backlinks: none
+  :local:
+
+********************
+C++ Coding Guideline
+********************
+- Follow `Google style for C++ <https://google.github.io/styleguide/cppguide.html>`_, with two exceptions:
+
+  * Each line of text may contain up to 100 characters.
+  * The use of C++ exceptions is allowed.
+
+- Use C++11 features such as smart pointers, braced initializers, lambda functions, and ``std::thread``.
+- Use Doxygen to document all the interface code.
+- We have a series of automatic checks to ensure that all of our codebase complies with the Google style. Before submitting your pull request, you are encouraged to run the style checks on your machine. See :ref:`running_checks_locally`.
+
+***********************
+Python Coding Guideline
+***********************
+- Follow `PEP 8: Style Guide for Python Code <https://www.python.org/dev/peps/pep-0008/>`_. We use PyLint to automatically enforce PEP 8 style across our Python codebase. Before submitting your pull request, you are encouraged to run PyLint on your machine. See :ref:`running_checks_locally`.
+- Docstrings should be in `NumPy docstring format <https://numpydoc.readthedocs.io/en/latest/format.html>`_.
+
+.. _running_checks_locally:
+
+******************
+R Coding Guideline
+******************
+
+Code Style
+==========
+- We follow Google's C++ Style guide for C++ code.
+
+  - This is mainly to be consistent with the rest of the project.
+  - Another reason is we will be able to check style automatically with a linter.
+
+- You can check the style of the code by typing the following command at root folder.
+
+  .. code-block:: bash
+
+    make rcpplint
+
+- When needed, you can disable the linter warning of certain line with ``// NOLINT(*)`` comments.
+- We use `roxygen <https://cran.r-project.org/web/packages/roxygen2/vignettes/roxygen2.html>`_ for documenting the R package.
+
+Rmarkdown Vignettes
+===================
+Rmarkdown vignettes are placed in `R-package/vignettes <https://github.com/dmlc/xgboost/tree/master/R-package/vignettes>`_.
+These Rmarkdown files are not compiled. We host the compiled version on `doc/R-package <https://github.com/dmlc/xgboost/tree/master/doc/R-package>`_.
+
+The following steps are followed to add a new Rmarkdown vignettes:
+
+- Add the original rmarkdown to ``R-package/vignettes``.
+- Modify ``doc/R-package/Makefile`` to add the markdown files to be build.
+- Clone the `dmlc/web-data <https://github.com/dmlc/web-data>`_ repo to folder ``doc``.
+- Now type the following command on ``doc/R-package``:
+
+  .. code-block:: bash
+
+    make the-markdown-to-make.md
+
+- This will generate the markdown, as well as the figures in ``doc/web-data/xgboost/knitr``.
+- Modify the ``doc/R-package/index.md`` to point to the generated markdown.
+- Add the generated figure to the ``dmlc/web-data`` repo.
+
+  - If you already cloned the repo to doc, this means ``git add``
+
+- Create PR for both the markdown and ``dmlc/web-data``.
+- You can also build the document locally by typing the following command at the ``doc`` directory:
+
+  .. code-block:: bash
+
+    make html
+
+The reason we do this is to avoid exploded repo size due to generated images.
+
+R package versioning
+====================
+See :ref:`release`.
+
+Registering native routines in R
+================================
+According to `R extension manual <https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Registering-native-routines>`_,
+it is good practice to register native routines and to disable symbol search. When any changes or additions are made to the
+C++ interface of the R package, please make corresponding changes in ``src/init.c`` as well.
+
+*********************************
+Running Formatting Checks Locally
+*********************************
+
+Once you submit a pull request to `dmlc/xgboost <https://github.com/dmlc/xgboost>`_, we perform
+two automatic checks to enforce coding style conventions. To expedite the code review process, you are encouraged to run the checks locally on your machine prior to submitting your pull request.
+
+Linter
+======
+We use `pylint <https://github.com/PyCQA/pylint>`_ and `cpplint <https://github.com/cpplint/cpplint>`_ to enforce style convention and find potential errors. Linting is especially useful for Python, as we can catch many errors that would have otherwise occured at run-time.
+
+To run this check locally, run the following command from the top level source tree:
+
+.. code-block:: bash
+
+  cd /path/to/xgboost/
+  make lint
+
+This command requires the Python packages pylint and cpplint.
+
+Clang-tidy
+==========
+`Clang-tidy <https://clang.llvm.org/extra/clang-tidy/>`_ is an advance linter for C++ code, made by the LLVM team. We use it to conform our C++ codebase to modern C++ practices and conventions.
+
+To run this check locally, run the following command from the top level source tree:
+
+.. code-block:: bash
+
+  cd /path/to/xgboost/
+  python3 tests/ci_build/tidy.py
+
+Also, the script accepts two optional integer arguments, namely ``--cpp`` and ``--cuda``. By default they are both set to 1, meaning that both C++ and CUDA code will be checked. If the CUDA toolkit is not installed on your machine, you'll encounter an error. To exclude CUDA source from linting, use:
+
+.. code-block:: bash
+
+  cd /path/to/xgboost/
+  python3 tests/ci_build/tidy.py --cuda=0
+
+Similarly, if you want to exclude C++ source from linting:
+
+.. code-block:: bash
+
+  cd /path/to/xgboost/
+  python3 tests/ci_build/tidy.py --cpp=0
+

doc/contrib/community.rst

@@ -0,0 +1,33 @@
+.. _community_guide:
+
+XGBoost Community Guideline
+===========================
+
+XGBoost adopts the Apache style model and governs by merit. We believe that it is important to create an inclusive community where everyone can use, contribute to, and influence the direction of the project. See `CONTRIBUTORS.md <https://github.com/dmlc/xgboost/blob/master/CONTRIBUTORS.md>`_ for the current list of contributors.
+
+
+
+General Development Process
+---------------------------
+Everyone in the community is welcomed to send patches, documents, and propose new directions to the project. The key guideline here is to enable everyone in the community to get involved and participate the decision and development.  When major changes are proposed, an RFC should be sent to allow discussion by the community. We encourage public discussion, archivable channels such as issues and discuss forum, so that everyone in the community can participate and review the process later.
+
+Code reviews are one of the key ways to ensure the quality of the code. High-quality code reviews prevent technical debt for long-term and are crucial to the success of the project. A pull request needs to be reviewed before it gets merged. A committer who has the expertise of the corresponding area would moderate the pull request and the merge the code when it is ready. The corresponding committer could request multiple reviewers who are familiar with the area of the code. We encourage contributors to request code reviews themselves and help review each other's code -- remember everyone is volunteering their time to the community, high-quality code review itself costs as much as the actual code contribution, you could get your code quickly reviewed if you do others the same favor.
+
+The community should strive to reach a consensus on technical decisions through discussion. We expect committers and PMCs to moderate technical discussions in a diplomatic way, and provide suggestions with clear technical reasoning when necessary.
+
+
+
+Committers
+----------
+Committers are individuals who are granted the write access to the project. A committer is usually responsible for a certain area or several areas of the code where they oversee the code review process. The area of contribution can take all forms, including code contributions and code reviews, documents, education, and outreach. Committers are essential for a high quality and healthy project. The community actively look for new committers from contributors. Here is a list of useful traits that help the community to recognize potential committers:
+
+- Sustained contribution to the project, demonstrated by discussion over RFCs, code reviews and proposals of new features, and other development activities. Being familiar with, and being able to take ownership on one or several areas of the project.
+- Quality of contributions: High-quality, readable code contributions indicated by pull requests that can be merged without a substantial code review.  History of creating clean, maintainable code and including good test cases. Informative code reviews to help other contributors that adhere to a good standard.
+- Community involvement: active participation in the discussion forum, promote the projects via tutorials, talks and outreach. We encourage committers to collaborate broadly, e.g. do code reviews and discuss designs with community members that they do not interact physically.
+
+The Project Management Committee(PMC) consists group of active committers that moderate the discussion, manage the project release, and proposes new committer/PMC members. Potential candidates are usually proposed via an internal discussion among PMCs, followed by a consensus approval, i.e. least 3 +1 votes, and no vetoes. Any veto must be accompanied by reasoning. PMCs should serve the community by upholding the community practices and guidelines XGBoost a better community for everyone. PMCs should strive to only nominate new candidates outside of their own organization.
+
+
+Reviewers
+---------
+Reviewers are individuals who actively contributed to the project and are willing to participate in the code review of new contributions. We identify reviewers from active contributors. The committers should explicitly solicit reviews from reviewers.  High-quality code reviews prevent technical debt for long-term and are crucial to the success of the project. A pull request to the project has to be reviewed by at least one reviewer in order to be merged.

doc/contrib/docs.rst

@@ -0,0 +1,30 @@
+##########################
+Documentation and Examples
+##########################
+
+**Contents**
+
+.. contents::
+  :backlinks: none
+  :local:
+
+*********
+Documents
+*********
+* Documentation is built using `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
+* Each document is written in `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_.
+* You can build document locally to see the effect, by running
+
+.. code-block:: bash
+
+  make html
+
+inside the ``doc/`` directory.
+
+********
+Examples
+********
+* Use cases and examples will be in `demo <https://github.com/dmlc/xgboost/tree/master/demo>`_.
+* We are super excited to hear about your story, if you have blogposts,
+  tutorials code solutions using XGBoost, please tell us and we will add
+  a link in the example pages.

doc/contrib/git_guide.rst

@@ -0,0 +1,76 @@
+###################
+Git Workflow Howtos
+###################
+
+**Contents**
+
+.. contents::
+  :backlinks: none
+  :local:
+
+***********************************
+How to resolve conflict with master
+***********************************
+
+- First rebase to most recent master
+
+  .. code-block:: bash
+
+    # The first two steps can be skipped after you do it once.
+    git remote add upstream https://github.com/dmlc/xgboost
+    git fetch upstream
+    git rebase upstream/master
+
+- The git may show some conflicts it cannot merge, say ``conflicted.py``.
+
+  - Manually modify the file to resolve the conflict.
+  - After you resolved the conflict, mark it as resolved by
+
+    .. code-block:: bash
+
+      git add conflicted.py
+
+- Then you can continue rebase by
+
+  .. code-block:: bash
+
+    git rebase --continue
+
+- Finally push to your fork, you may need to force push here.
+
+  .. code-block:: bash
+
+    git push --force
+
+****************************************
+How to combine multiple commits into one
+****************************************
+Sometimes we want to combine multiple commits, especially when later commits are only fixes to previous ones,
+to create a PR with set of meaningful commits. You can do it by following steps.
+
+- Before doing so, configure the default editor of git if you haven't done so before.
+
+  .. code-block:: bash
+
+    git config core.editor the-editor-you-like
+
+- Assume we want to merge last 3 commits, type the following commands
+
+  .. code-block:: bash
+
+    git rebase -i HEAD~3
+
+- It will pop up an text editor. Set the first commit as ``pick``, and change later ones to ``squash``.
+- After you saved the file, it will pop up another text editor to ask you modify the combined commit message.
+- Push the changes to your fork, you need to force push.
+
+  .. code-block:: bash
+
+    git push --force
+
+*************************************
+What is the consequence of force push
+*************************************
+The previous two tips requires force push, this is because we altered the path of the commits.
+It is fine to force push to your own fork, as long as the commits changed are only yours.
+

doc/contrib/index.rst

@@ -0,0 +1,28 @@
+#####################
+Contribute to XGBoost
+#####################
+
+XGBoost has been developed by community members. Everyone is welcome to contribute. We value all forms of contributions, including, but not limited to:
+
+* Code reviews for pull requests
+* Documentation and usage examples
+* Community participation in forums and issues
+* Code readability and developer guide
+
+  - We welcome contributions that add code comments to improve readability.
+  - We also welcome contributions to docs to explain the design choices of the XGBoost internals.
+
+* Test cases to make the codebase more robust.
+* Tutorials, blog posts, talks that promote the project.
+
+Here are guidelines for contributing to various aspect of the XGBoost project:
+
+.. toctree::
+  :maxdepth: 2
+
+  Community Guideline <community>
+  coding_guide
+  unit_tests
+  Docs and Examples <docs>
+  git_guide
+  release

doc/contrib/release.rst

@@ -0,0 +1,13 @@
+.. _release:
+
+XGBoost Release Policy
+=======================
+
+Versioning Policy
+---------------------------
+
+Starting from XGBoost 1.0.0, each XGBoost release will be versioned as [MAJOR].[FEATURE].[MAINTENANCE]
+
+* MAJOR: We gurantee the API compatibility across releases with the same major version number. We expect to have a 1+ years development period for a new MAJOR release version.
+* FEATURE: We ship new features, improvements and bug fixes through feature releases. The cycle length of a feature is decided by the size of feature roadmap. The roadmap is decided right after the previous release.
+* MAINTENANCE: Maintenance version only contains bug fixes. This type of release only occurs when we found significant correctness and/or performance bugs and barrier for users to upgrade to a new version of XGBoost smoothly.

doc/contrib/unit_tests.rst

@@ -0,0 +1,179 @@
+########################
+Adding and running tests
+########################
+
+A high-quality suite of tests is crucial in ensuring correctness and robustness of the codebase. Here, we provide instructions how to run unit tests, and also how to add a new one.
+
+**Contents**
+
+.. contents::
+  :backlinks: none
+  :local:
+
+**********************
+Adding a new unit test
+**********************
+
+Python package: pytest
+======================
+Add your test under the directory `tests/python/ <https://github.com/dmlc/xgboost/tree/master/tests/python>`_ or `tests/python-gpu/ <https://github.com/dmlc/xgboost/tree/master/tests/python-gpu>`_ (if you are testing GPU code). Refer to `the PyTest tutorial <https://docs.pytest.org/en/latest/getting-started.html>`_ to learn how to write tests for Python code.
+
+You may try running your test by following instructions in :ref:`this section <running_pytest>`.
+
+C++: Google Test
+================
+Add your test under the directory `tests/cpp/ <https://github.com/dmlc/xgboost/tree/master/tests/cpp>`_. Refer to `this excellent tutorial on using Google Test <https://developer.ibm.com/articles/au-googletestingframework/>`_.
+
+You may try running your test by following instructions in :ref:`this section <running_gtest>`. Note. Google Test version 1.8.1 or later is required.
+
+JVM packages: JUnit / scalatest
+===============================
+The JVM packages for XGBoost (XGBoost4J / XGBoost4J-Spark) use `the Maven Standard Directory Layout <https://maven.apache.org/guides/introduction/introduction-to-the-standard-directory-layout.html>`_. Specifically, the tests for the JVM packages are located in the following locations:
+
+* `jvm-packages/xgboost4j/src/test/ <https://github.com/dmlc/xgboost/tree/master/jvm-packages/xgboost4j/src/test>`_
+* `jvm-packages/xgboost4j-spark/src/test/ <https://github.com/dmlc/xgboost/tree/master/jvm-packages/xgboost4j-spark/src/test>`_
+
+To write a test for Java code, see `JUnit 5 tutorial <https://junit.org/junit5/docs/current/user-guide/>`_.
+To write a test for Scala, see `Scalatest tutorial <http://www.scalatest.org/user_guide/writing_your_first_test>`_.
+
+You may try running your test by following instructions in :ref:`this section <running_jvm_tests>`.
+
+R package: testthat
+===================
+Add your test under the directory `R-package/tests/testthat <https://github.com/dmlc/xgboost/tree/master/R-package/tests/testthat>`_. Refer to `this excellent tutorial on testthat <https://kbroman.org/pkg_primer/pages/tests.html>`_.
+
+You may try running your test by following instructions in :ref:`this section <running_r_tests>`.
+
+**************************
+Running Unit Tests Locally
+**************************
+
+.. _running_r_tests:
+
+R package
+=========
+Run
+
+.. code-block:: bash
+
+  make Rcheck
+
+at the root of the project directory.
+
+.. _running_jvm_tests:
+
+JVM packages
+============
+As part of the building process, tests are run:
+
+.. code-block:: bash
+
+  mvn package
+
+.. _running_pytest:
+
+Python package: pytest
+======================
+
+To run Python unit tests, first install `pytest <https://docs.pytest.org/en/latest/contents.html>`_ package:
+
+.. code:: bash
+
+  pip3 install pytest
+
+Then compile XGBoost according to instructions in :ref:`build_shared_lib`. Finally, invoke pytest at the project root directory:
+
+.. code:: bash
+
+  # Tell Python where to find XGBoost module
+  export PYTHONPATH=./python-package
+  pytest -v -s --fulltrace tests/python
+
+In addition, to test CUDA code, run:
+
+.. code:: bash
+
+  # Tell Python where to find XGBoost module
+  export PYTHONPATH=./python-package
+  pytest -v -s --fulltrace tests/python-gpu
+
+(For this step, you should have compiled XGBoost with CUDA enabled.)
+
+.. _running_gtest:
+
+C++: Google Test
+================
+
+To build and run C++ unit tests enable tests while running CMake:
+
+.. code-block:: bash
+
+  mkdir build
+  cd build
+  cmake -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON  ..
+  make
+  make test
+
+To enable tests for CUDA code, add ``-DUSE_CUDA=ON`` and ``-DUSE_NCCL=ON`` (CUDA toolkit required):
+
+.. code-block:: bash
+
+  mkdir build
+  cd build
+  cmake -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DUSE_CUDA=ON -DUSE_NCCL=ON ..
+  make
+  make test
+
+One can also run all unit test using ctest tool which provides higher flexibility. For example:
+
+.. code-block:: bash
+
+  ctest --verbose
+
+***********************************************
+Sanitizers: Detect memory errors and data races
+***********************************************
+
+By default, sanitizers are bundled in GCC and Clang/LLVM. One can enable
+sanitizers with GCC >= 4.8 or LLVM >= 3.1, But some distributions might package
+sanitizers separately.  Here is a list of supported sanitizers with
+corresponding library names:
+
+- Address sanitizer: libasan
+- Leak sanitizer:    liblsan
+- Thread sanitizer:  libtsan
+
+Memory sanitizer is exclusive to LLVM, hence not supported in XGBoost.
+
+How to build XGBoost with sanitizers
+====================================
+One can build XGBoost with sanitizer support by specifying -DUSE_SANITIZER=ON.
+By default, address sanitizer and leak sanitizer are used when you turn the
+USE_SANITIZER flag on.  You can always change the default by providing a
+semicolon separated list of sanitizers to ENABLED_SANITIZERS.  Note that thread
+sanitizer is not compatible with the other two sanitizers.
+
+.. code-block:: bash
+
+  cmake -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak" /path/to/xgboost
+
+By default, CMake will search regular system paths for sanitizers, you can also
+supply a specified SANITIZER_PATH.
+
+.. code-block:: bash
+
+  cmake -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak" \
+  -DSANITIZER_PATH=/path/to/sanitizers /path/to/xgboost
+
+How to use sanitizers with CUDA support
+=======================================
+Runing XGBoost on CUDA with address sanitizer (asan) will raise memory error.
+To use asan with CUDA correctly, you need to configure asan via ASAN_OPTIONS
+environment variable:
+
+.. code-block:: bash
+
+  ASAN_OPTIONS=protect_shadow_gap=0 ${BUILD_DIR}/testxgboost
+
+For details, please consult `official documentation <https://github.com/google/sanitizers/wiki>`_ for sanitizers.
+

doc/contribute.rst

@@ -1,290 +0,0 @@
-#####################
-Contribute to XGBoost
-#####################
-XGBoost has been developed and used by a group of active community members.
-Everyone is more than welcome to contribute. It is a way to make the project better and more accessible to more users.
-
-- Please add your name to `CONTRIBUTORS.md <https://github.com/dmlc/xgboost/blob/master/CONTRIBUTORS.md>`_ after your patch has been merged.
-- Please also update `NEWS.md <https://github.com/dmlc/xgboost/blob/master/NEWS.md>`_ to add note on your changes to the API or XGBoost documentation.
-
-**Guidelines**
-
-* `Submit Pull Request`_
-* `Git Workflow Howtos`_
-
-  - `How to resolve conflict with master`_
-  - `How to combine multiple commits into one`_
-  - `What is the consequence of force push`_
-
-* `Documents`_
-* `Testcases`_
-* `Sanitizers`_
-* `clang-tidy`_
-* `Examples`_
-* `Core Library`_
-* `Python Package`_
-* `R Package`_
-
-*******************
-Submit Pull Request
-*******************
-
-* Before submit, please rebase your code on the most recent version of master, you can do it by
-
-  .. code-block:: bash
-
-    git remote add upstream https://github.com/dmlc/xgboost
-    git fetch upstream
-    git rebase upstream/master
-
-* If you have multiple small commits,
-  it might be good to merge them together(use git rebase then squash) into more meaningful groups.
-* Send the pull request!
-
-  - Fix the problems reported by automatic checks
-  - If you are contributing a new module, consider add a testcase in `tests <https://github.com/dmlc/xgboost/tree/master/tests>`_.
-
-*******************
-Git Workflow Howtos
-*******************
-
-How to resolve conflict with master
-===================================
-- First rebase to most recent master
-
-  .. code-block:: bash
-
-    # The first two steps can be skipped after you do it once.
-    git remote add upstream https://github.com/dmlc/xgboost
-    git fetch upstream
-    git rebase upstream/master
-
-- The git may show some conflicts it cannot merge, say ``conflicted.py``.
-
-  - Manually modify the file to resolve the conflict.
-  - After you resolved the conflict, mark it as resolved by
-
-    .. code-block:: bash
-
-      git add conflicted.py
-
-- Then you can continue rebase by
-
-  .. code-block:: bash
-
-    git rebase --continue
-
-- Finally push to your fork, you may need to force push here.
-
-  .. code-block:: bash
-
-    git push --force
-
-How to combine multiple commits into one
-========================================
-Sometimes we want to combine multiple commits, especially when later commits are only fixes to previous ones,
-to create a PR with set of meaningful commits. You can do it by following steps.
-
-- Before doing so, configure the default editor of git if you haven't done so before.
-
-  .. code-block:: bash
-
-    git config core.editor the-editor-you-like
-
-- Assume we want to merge last 3 commits, type the following commands
-
-  .. code-block:: bash
-
-    git rebase -i HEAD~3
-
-- It will pop up an text editor. Set the first commit as ``pick``, and change later ones to ``squash``.
-- After you saved the file, it will pop up another text editor to ask you modify the combined commit message.
-- Push the changes to your fork, you need to force push.
-
-  .. code-block:: bash
-
-    git push --force
-
-What is the consequence of force push
-=====================================
-The previous two tips requires force push, this is because we altered the path of the commits.
-It is fine to force push to your own fork, as long as the commits changed are only yours.
-
-*********
-Documents
-*********
-* Documentation is built using sphinx.
-* Each document is written in `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_.
-* You can build document locally to see the effect.
-
-*********
-Testcases
-*********
-* All the testcases are in `tests <https://github.com/dmlc/xgboost/tree/master/tests>`_.
-* We use python nose for python test cases.
-
-**********
-Sanitizers
-**********
-
-By default, sanitizers are bundled in GCC and Clang/LLVM. One can enable
-sanitizers with GCC >= 4.8 or LLVM >= 3.1, But some distributions might package
-sanitizers separately.  Here is a list of supported sanitizers with
-corresponding library names:
-
-- Address sanitizer: libasan
-- Leak sanitizer:    liblsan
-- Thread sanitizer:  libtsan
-
-Memory sanitizer is exclusive to LLVM, hence not supported in XGBoost.
-
-How to build XGBoost with sanitizers
-====================================
-One can build XGBoost with sanitizer support by specifying -DUSE_SANITIZER=ON.
-By default, address sanitizer and leak sanitizer are used when you turn the
-USE_SANITIZER flag on.  You can always change the default by providing a
-semicolon separated list of sanitizers to ENABLED_SANITIZERS.  Note that thread
-sanitizer is not compatible with the other two sanitizers.
-
-  .. code-block:: bash
-
-    cmake -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak" /path/to/xgboost
-
-By default, CMake will search regular system paths for sanitizers, you can also
-supply a specified SANITIZER_PATH.
-
-  .. code-block:: bash
-
-    cmake -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak" \
-    -DSANITIZER_PATH=/path/to/sanitizers /path/to/xgboost
-
-How to use sanitizers with CUDA support
-=======================================
-Runing XGBoost on CUDA with address sanitizer (asan) will raise memory error.
-To use asan with CUDA correctly, you need to configure asan via ASAN_OPTIONS
-environment variable:
-
-  .. code-block:: bash
-
-    ASAN_OPTIONS=protect_shadow_gap=0 ${BUILD_DIR}/testxgboost
-
-For details, please consult `official documentation <https://github.com/google/sanitizers/wiki>`_ for sanitizers.
-
-**********
-clang-tidy
-**********
-To run clang-tidy on both C++ and CUDA source code,  run the following command
-from the top level source tree:
-
-  .. code-black:: bash
-    cd /path/to/xgboost/
-    python3 tests/ci_build/tidy.py --gtest-path=/path/to/google-test
-
-The script requires the full path of Google Test library via the ``--gtest-path`` argument.
-
-Also, the script accepts two optional integer arguments, namely ``--cpp`` and ``--cuda``.
-By default they are both set to 1.  If you want to exclude CUDA source from
-linting, use:
-
-  .. code-black:: bash
-    cd /path/to/xgboost/
-    python3 tests/ci_build/tidy.py --cuda=0
-
-Similarly, if you want to exclude C++ source from linting:
-
-  .. code-black:: bash
-    cd /path/to/xgboost/
-    python3 tests/ci_build/tidy.py --cpp=0
-
-********
-Examples
-********
-* Usecases and examples will be in `demo <https://github.com/dmlc/xgboost/tree/master/demo>`_.
-* We are super excited to hear about your story, if you have blogposts,
-  tutorials code solutions using XGBoost, please tell us and we will add
-  a link in the example pages.
-
-************
-Core Library
-************
-- Follow `Google style for C++ <https://google.github.io/styleguide/cppguide.html>`_.
-- Use C++11 features such as smart pointers, braced initializers, lambda functions, and ``std::thread``.
-- We use Doxygen to document all the interface code.
-- You can reproduce the linter checks by running ``make lint``
-
-**************
-Python Package
-**************
-- Always add docstring to the new functions in numpydoc format.
-- You can reproduce the linter checks by typing ``make lint``
-
-*********
-R Package
-*********
-
-Code Style
-==========
-- We follow Google's C++ Style guide for C++ code.
-
-  - This is mainly to be consistent with the rest of the project.
-  - Another reason is we will be able to check style automatically with a linter.
-
-- You can check the style of the code by typing the following command at root folder.
-
-  .. code-block:: bash
-
-    make rcpplint
-
-- When needed, you can disable the linter warning of certain line with ```// NOLINT(*)``` comments.
-- We use `roxygen <https://cran.r-project.org/web/packages/roxygen2/vignettes/roxygen2.html>`_ for documenting the R package.
-
-Rmarkdown Vignettes
-===================
-Rmarkdown vignettes are placed in `R-package/vignettes <https://github.com/dmlc/xgboost/tree/master/R-package/vignettes>`_.
-These Rmarkdown files are not compiled. We host the compiled version on `doc/R-package <https://github.com/dmlc/xgboost/tree/master/doc/R-package>`_.
-
-The following steps are followed to add a new Rmarkdown vignettes:
-
-- Add the original rmarkdown to ``R-package/vignettes``.
-- Modify ``doc/R-package/Makefile`` to add the markdown files to be build.
-- Clone the `dmlc/web-data <https://github.com/dmlc/web-data>`_ repo to folder ``doc``.
-- Now type the following command on ``doc/R-package``:
-
-  .. code-block:: bash
-
-    make the-markdown-to-make.md
-
-- This will generate the markdown, as well as the figures in ``doc/web-data/xgboost/knitr``.
-- Modify the ``doc/R-package/index.md`` to point to the generated markdown.
-- Add the generated figure to the ``dmlc/web-data`` repo.
-
-  - If you already cloned the repo to doc, this means ``git add``
-
-- Create PR for both the markdown  and ``dmlc/web-data``.
-- You can also build the document locally by typing the following command at the ``doc`` directory:
-
-  .. code-block:: bash
-
-    make html
-
-The reason we do this is to avoid exploded repo size due to generated images.
-
-R package versioning
-====================
-Since version 0.6.4.3, we have adopted a versioning system that uses x.y.z (or ``core_major.core_minor.cran_release``)
-format for CRAN releases and an x.y.z.p (or ``core_major.core_minor.cran_release.patch``) format for development patch versions.
-This approach is similar to the one described in Yihui Xie's
-`blog post on R Package Versioning <https://yihui.name/en/2013/06/r-package-versioning/>`_,
-except we need an additional field to accomodate the x.y core library version.
-
-Each new CRAN release bumps up the 3rd field, while developments in-between CRAN releases
-would be marked by an additional 4th field on the top of an existing CRAN release version.
-Some additional consideration is needed when the core library version changes.
-E.g., after the core changes from 0.6 to 0.7, the R package development version would become 0.7.0.1, working towards
-a 0.7.1 CRAN release. The 0.7.0 would not be released to CRAN, unless it would require almost no additional development.
-
-Registering native routines in R
-================================
-According to `R extension manual <https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Registering-native-routines>`_,
-it is good practice to register native routines and to disable symbol search. When any changes or additions are made to the
-C++ interface of the R package, please make corresponding changes in ``src/init.c`` as well.

doc/get_started.rst

@@ -24,7 +24,7 @@ Python
   dtrain = xgb.DMatrix('demo/data/agaricus.txt.train')
   dtest = xgb.DMatrix('demo/data/agaricus.txt.test')
   # specify parameters via map
-  param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
+  param = {'max_depth':2, 'eta':1, 'objective':'binary:logistic' }
   num_round = 2
   bst = xgb.train(param, dtrain, num_round)
   # make prediction